feat: preserve solo runs across production restarts

This commit is contained in:
jwinterm
2026-07-22 15:05:49 -04:00
parent b2772ff73b
commit 80d7f35e1f
37 changed files with 1816 additions and 160 deletions
+2 -2
View File
@@ -20,8 +20,8 @@ const SOURCE_SOFTWARE = Object.freeze({
license: 'MIT',
rightsNotice: 'Permission to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies is granted under the MIT License only if the copyright and permission notice is included in all copies or substantial portions.',
warrantyNotice: 'The software is provided “AS IS”, without warranty of any kind, express or implied, to the fullest extent stated in the MIT License.',
legalAdviceNotice: 'The software, documentation, examples, and disclosures are not legal advice, a licence, or a compliance approval.',
thirdPartyNotice: 'A third-party or self-hosted operator is solely responsible for its deployment, product design, legal compliance, funds, players, claims, and support. Such Software does not operate, supervise, endorse, or accept responsibility for third-party deployments.',
legalAdviceNotice: 'Documentation, examples, and product disclosures are informational only; they are not legal advice or a determination that any deployment complies with applicable law.',
thirdPartyNotice: 'A third-party or self-hosted operator is solely responsible for its deployment, product design, legal compliance, funds, players, claims, and support. The MIT License does not make that deployment a Such Software service or authorize it to claim Such Software sponsorship or endorsement. Such Software does not operate, supervise, endorse, or accept responsibility for third-party deployments.',
operatedBoundaryNotice: 'Such Software operates only play.wowne.ro and monerogue.app, and only within the product scopes stated for those services. It does not operate or endorse deployments made from this source by anyone else.'
});
+30 -5
View File
@@ -42,7 +42,10 @@ const DUNGEON_CONFIGS = {
* Dungeon generation utilities
*/
class DungeonGenerator {
static get GENERATOR_VERSION() { return 'dungeon-generator-v1'; }
static get GENERATOR_VERSION() { return 'dungeon-generator-v2'; }
static get SUPPORTED_GENERATOR_VERSIONS() {
return Object.freeze(['dungeon-generator-v1', 'dungeon-generator-v2']);
}
static get FINGERPRINT_VERSION() { return 1; }
// Get the configuration object with difficulty settings applied
@@ -137,6 +140,10 @@ class DungeonGenerator {
};
const config = { ...defaultOptions, ...options };
const generatorVersion = config.generatorVersion || this.GENERATOR_VERSION;
if (!this.SUPPORTED_GENERATOR_VERSIONS.includes(generatorVersion)) {
throw new Error(`Unsupported dungeon generator version: ${generatorVersion}`);
}
if (CONSOLE_LOGGING) {
console.log(`[DungeonGenerator] Generating dungeon with effective torchDensity: ${config.torchDensity}`);
}
@@ -157,15 +164,22 @@ class DungeonGenerator {
const attemptSeed = (baseSeedInt != null) ? baseSeedInt + attempt : null;
const dungeon = this._generateOnce(width, height, config, rng, attemptSeed);
last = dungeon;
if (this.isReachable(dungeon.map, dungeon.entrance, dungeon.exit, config)) {
const reachable = this.isReachable(dungeon.map, dungeon.entrance, dungeon.exit, config);
const objectivesValid = generatorVersion === 'dungeon-generator-v1'
|| this.hasDistinctReachableObjectives(dungeon, config);
if (reachable && objectivesValid) {
return dungeon;
}
if (CONSOLE_LOGGING) {
console.warn(`[DungeonGenerator] attempt ${attempt}: exit unreachable from entrance, regenerating`);
console.warn(`[DungeonGenerator] attempt ${attempt}: invalid reachability/objectives, regenerating`);
}
}
console.error(`[DungeonGenerator] Could not generate a reachable dungeon after ${MAX_ATTEMPTS} attempts; returning last candidate.`);
return last;
if (generatorVersion === 'dungeon-generator-v1') {
// Exact historical behavior for already-recorded v1 proofs.
console.error(`[DungeonGenerator] Could not generate a reachable dungeon after ${MAX_ATTEMPTS} attempts; returning last candidate.`);
return last;
}
throw new Error(`Could not generate a playable dungeon after ${MAX_ATTEMPTS} deterministic attempts`);
}
/**
@@ -278,6 +292,17 @@ class DungeonGenerator {
}
return false;
}
/** V2 requires three distinct, reachable objectives so a run cannot start already won/looted. */
static hasDistinctReachableObjectives(dungeon, config) {
const points = [dungeon?.entrance, dungeon?.exit, dungeon?.treasure];
if (points.some(point => !Array.isArray(point) || point.length !== 2
|| !Number.isInteger(point[0]) || !Number.isInteger(point[1]))) return false;
const keys = points.map(([x, y]) => `${x},${y}`);
if (new Set(keys).size !== points.length) return false;
return this.isReachable(dungeon.map, dungeon.entrance, dungeon.exit, config)
&& this.isReachable(dungeon.map, dungeon.entrance, dungeon.treasure, config);
}
// Enhanced map creation with floor variations and torch placement
static enhanceMapWithVariations(basicMap, config) {
+3 -3
View File
@@ -26,7 +26,7 @@ function attachDungeonVerification(result, {
? context.generationOptions
: {};
const recordedGenerator = generatorVersion || context.generatorVersion || DungeonGenerator.GENERATOR_VERSION;
if (recordedGenerator !== DungeonGenerator.GENERATOR_VERSION) {
if (!DungeonGenerator.SUPPORTED_GENERATOR_VERSIONS.includes(recordedGenerator)) {
throw Object.assign(new Error('unsupported_generator_version'), { code: 'UNSUPPORTED_GENERATOR' });
}
@@ -53,14 +53,14 @@ function attachDungeonVerification(result, {
const levels = manifest.map((entry) => {
const depth = Number(entry.depth);
const entryGenerator = entry.generatorVersion || recordedGenerator;
if (entryGenerator !== DungeonGenerator.GENERATOR_VERSION) {
if (!DungeonGenerator.SUPPORTED_GENERATOR_VERSIONS.includes(entryGenerator)) {
throw Object.assign(new Error('unsupported_generator_version'), { code: 'UNSUPPORTED_GENERATOR' });
}
const fingerprintVersion = Number(entry.fingerprintVersion || context.fingerprintVersion || 1);
const dungeon = DungeonGenerator.regenerateFromSeed(
levelSeed(effectiveSeed, depth),
context.cryptoType || process.env.CRYPTO_TYPE || 'WOW',
options
{ ...options, generatorVersion: entryGenerator }
);
if (depth < maxDepth) dungeon.treasure = null;
const fingerprint = DungeonGenerator.layoutFingerprint(dungeon, fingerprintVersion);
+66 -5
View File
@@ -9,6 +9,10 @@ const { createGameProof, getPreGameCommitment, getPostGameReveal, createSeededRN
// Environment-based console logging control
const CONSOLE_LOGGING = process.env.NODE_ENV === 'debug' || process.env.NODE_ENV === 'development';
// Only this module can mint the option that selects a historical generator. Public/client game
// options may contain a same-named string, but the constructor deliberately discards it. Durable
// restart restoration supplies the database-validated version through the symbol-backed factory.
const TRUSTED_GENERATOR_VERSION = Symbol('trustedGeneratorVersion');
class Game {
/**
@@ -22,7 +26,17 @@ class Game {
const fairnessProof = suppliedOptions.fairnessProof || null;
// Internal proof material (including the secret server seed) must never ride into dungeon
// options, snapshots, logs, or the persisted public generation context.
const { fairnessProof: _privateFairnessProof, clientSeed: optionClientSeed, ...dungeonOptions } = suppliedOptions;
const {
fairnessProof: _privateFairnessProof,
clientSeed: optionClientSeed,
generatorVersion: _untrustedGeneratorVersion,
[TRUSTED_GENERATOR_VERSION]: trustedGeneratorVersion,
...dungeonOptions
} = suppliedOptions;
this.generatorVersion = trustedGeneratorVersion || DungeonGenerator.GENERATOR_VERSION;
if (!DungeonGenerator.SUPPORTED_GENERATOR_VERSIONS.includes(this.generatorVersion)) {
throw new Error(`Unsupported trusted dungeon generator version: ${this.generatorVersion}`);
}
this.user = user;
this.id = uuidv4();
this.socketId = socketId;
@@ -85,7 +99,7 @@ class Game {
// Bind the proof to every level in the committed run. Deeper layouts are independently
// regenerated from their per-depth seeds now, before play, so a crash or early death still
// leaves a complete, immutable verification manifest for the advertised descent.
this.gameProof.generatorVersion = DungeonGenerator.GENERATOR_VERSION;
this.gameProof.generatorVersion = this.generatorVersion;
this.gameProof.context = this.getProofContext();
this.gameProof.layoutFingerprints = this._buildLayoutFingerprintManifest();
// Backward-compatible level-one alias used by older verification clients/rows.
@@ -102,14 +116,17 @@ class Game {
dungeon = DungeonGenerator.regenerateFromSeed(
levelSeed(this.gameProof.seed, depth),
this.gameConfig.cryptoType,
this.gameProof.context.generationOptions
{
...this.gameProof.context.generationOptions,
generatorVersion: this.generatorVersion
}
);
if (depth < this.maxDepth) dungeon.treasure = null;
}
manifest.push({
depth,
fingerprintVersion: DungeonGenerator.FINGERPRINT_VERSION,
generatorVersion: DungeonGenerator.GENERATOR_VERSION,
generatorVersion: this.generatorVersion,
fingerprint: DungeonGenerator.layoutFingerprint(dungeon)
});
}
@@ -127,7 +144,7 @@ class Game {
return {
cryptoType: this.gameConfig.cryptoType || process.env.CRYPTO_TYPE || 'WOW',
maxDepth: this.maxDepth,
generatorVersion: DungeonGenerator.GENERATOR_VERSION,
generatorVersion: this.generatorVersion,
fingerprintVersion: DungeonGenerator.FINGERPRINT_VERSION,
generationOptions
};
@@ -140,6 +157,14 @@ class Game {
return new Game(socketId, user, options);
}
/** @internal Rebuild a run using the generator version already trusted from PostgreSQL. */
static createRestoredStandardGame(socketId, user, options = {}, generatorVersion) {
return new Game(socketId, user, {
...options,
[TRUSTED_GENERATOR_VERSION]: generatorVersion
});
}
/**
* Static factory method to create a legacy-sized game
*/
@@ -208,6 +233,7 @@ class Game {
const seed = levelSeed(this.gameProof.seed, depth);
const dungeon = DungeonGenerator.generate(this.gameConfig.width, this.gameConfig.height, {
...this.gameConfig,
generatorVersion: this.generatorVersion,
rng: isFirst ? this.seededRNG : createSeededRNG(seed),
seedInt: isFirst ? this.seedInt : seedToInt(seed)
});
@@ -242,6 +268,41 @@ class Game {
const center = dungeon.rooms[idx].getCenter();
this.monster.moveTo(center[0], center[1]);
}
// ROT.Digger can legitimately produce <=2 rooms on the 30x15 easy profile. The old branch
// then left the monster at its constructor default (0,0), normally a wall, producing an
// effectively monsterless run. Also defend against a malformed room center. Choose the
// farthest passable non-objective tile deterministically; this consumes no RNG and dungeon
// generation/layout fingerprints remain byte-identical (monster execution is explicitly
// outside the published layout proof).
const primaryFloor = this.gameConfig.primaryFloor || "'1";
const secondaryFloor = this.gameConfig.secondaryFloor || "'2";
const monsterTilePassable = (x, y) => {
const tile = dungeon.map?.[y]?.[x];
return tile === primaryFloor || tile === secondaryFloor || tile === 0 || tile === '>' || tile === '$M';
};
const monsterSpawnValid = monsterTilePassable(this.monster.x, this.monster.y)
&& (this.monster.x !== this.player.x || this.monster.y !== this.player.y)
&& (!dungeon.exit || this.monster.x !== dungeon.exit[0] || this.monster.y !== dungeon.exit[1])
&& (!dungeon.treasure || this.monster.x !== dungeon.treasure[0] || this.monster.y !== dungeon.treasure[1]);
if (!monsterSpawnValid) {
let fallback = null;
let fallbackDistance = -1;
for (let y = 0; y < dungeon.map.length; y += 1) {
for (let x = 0; x < dungeon.map[y].length; x += 1) {
if (!monsterTilePassable(x, y)) continue;
if (x === this.player.x && y === this.player.y) continue;
if (dungeon.exit && x === dungeon.exit[0] && y === dungeon.exit[1]) continue;
if (dungeon.treasure && x === dungeon.treasure[0] && y === dungeon.treasure[1]) continue;
const distance = Math.abs(x - this.player.x) + Math.abs(y - this.player.y);
if (distance > fallbackDistance) {
fallback = [x, y];
fallbackDistance = distance;
}
}
}
if (!fallback) throw new Error('Dungeon has no valid monster spawn tile');
this.monster.moveTo(fallback[0], fallback[1]);
}
// Reset lighting + FOV for the new level (a fresh descent is unexplored).
this.lightingAndFov = new LightingAndFov(dungeon.map, dungeon.torches);
+32 -1
View File
@@ -30,6 +30,8 @@ class GameManager {
Number(settlementRetryMaxMs) || 30000
);
this._isShuttingDown = false;
this._admissionClosed = false;
this._creationsInFlight = new Set();
}
/**
@@ -53,6 +55,21 @@ class GameManager {
* @returns {Object} Created game instance
*/
async createGameForUser(user, gameType = 'standard', options = {}) {
if (this._admissionClosed) {
const error = new Error('Game admission is closed for server shutdown');
error.code = 'SERVER_SHUTTING_DOWN';
throw error;
}
const creation = this._createGameForUser(user, gameType, options);
this._creationsInFlight.add(creation);
try {
return await creation;
} finally {
this._creationsInFlight.delete(creation);
}
}
async _createGameForUser(user, gameType = 'standard', options = {}) {
// Settlement-pending games intentionally remain in activeGames. This is the last-line
// guard against starting/charging a replacement run while the previous terminal result
// is not yet durable (and also closes ordinary double-start races between handlers).
@@ -86,6 +103,20 @@ class GameManager {
return game;
}
/** Synchronously close the lowest-level solo creation gate. */
beginShutdown() {
this._admissionClosed = true;
this._isShuttingDown = true;
}
/** Wait for creations that crossed the gate before beginShutdown(). */
async drainAdmissions() {
while (this._creationsInFlight.size > 0) {
await Promise.allSettled(Array.from(this._creationsInFlight));
}
return { pending: 0 };
}
/**
* Handle game over scenarios with comprehensive cleanup and payouts
* @param {Object} socket - Socket instance (or fake socket with just id)
@@ -371,7 +402,7 @@ class GameManager {
* Returns unresolved count so the process shutdown path can log/escalate it truthfully.
*/
async shutdown({ timeoutMs = 4000, retryIntervalMs = 50 } = {}) {
this._isShuttingDown = true;
this.beginShutdown();
for (const settlement of this._pendingSettlements.values()) {
if (settlement.retryTimer) clearTimeout(settlement.retryTimer);
settlement.retryTimer = null;
+29 -1
View File
@@ -94,6 +94,8 @@ class GameModeManager {
this._batchPayoutTimer = null;
this._isBatchProcessing = false;
this._isShuttingDown = false;
this._gameAdmissionClosed = false;
this._gameStartAdmissions = new Set();
// The composition root replaces this with the dynamic startup/runtime financial
// reconciliation gate. Default-open keeps isolated free/test instances compatible.
this.financialAdmissionAllowed = () => true;
@@ -1502,7 +1504,32 @@ class GameModeManager {
* SocketHandlers and passed into Game before its dungeon is generated.
*/
async processGameStart(socketId, gameId, options = {}) {
return this._dispatchGameStart(socketId, gameId, options);
if (this._gameAdmissionClosed) {
return {
success: false,
code: 'SERVER_SHUTTING_DOWN',
reason: 'Game admission is temporarily closed while the server restarts.'
};
}
const admission = this._dispatchGameStart(socketId, gameId, options);
this._gameStartAdmissions.add(admission);
try {
return await admission;
} finally {
this._gameStartAdmissions.delete(admission);
}
}
/** Close new solo entry consumption synchronously at the restart boundary. */
beginGameAdmissionShutdown() {
this._gameAdmissionClosed = true;
}
async drainGameStartAdmissions() {
while (this._gameStartAdmissions.size > 0) {
await Promise.allSettled(Array.from(this._gameStartAdmissions));
}
return { pending: 0 };
}
async _dispatchGameStart(socketId, gameId, options = {}) {
@@ -2768,6 +2795,7 @@ class GameModeManager {
/** Stop accepting scheduled payout work and cancel any debounce that has not fired yet. */
shutdown() {
this._isShuttingDown = true;
this.beginGameAdmissionShutdown();
if (this._batchPayoutTimer) {
clearTimeout(this._batchPayoutTimer);
this._batchPayoutTimer = null;
+23 -1
View File
@@ -97,7 +97,7 @@ function createSeededRNG(seed) {
let state = seed;
let counter = 0;
return function() {
const rng = function() {
// Generate next state by hashing current state + counter
const hash = crypto.createHash('sha256')
.update(state + counter.toString())
@@ -109,6 +109,28 @@ function createSeededRNG(seed) {
const num = hash.readUInt32BE(0);
return num / 0x100000000;
};
// A graceful restart must resume at the exact next draw; resetting the generator would make
// post-restart monster behaviour diverge from the committed run. These deliberately narrow
// accessors keep the normal callable-RNG API intact while allowing a validated runtime
// snapshot to preserve/restore only the monotonically increasing counter.
Object.defineProperties(rng, {
getCounter: {
value: () => counter,
enumerable: false
},
setCounter: {
value: (nextCounter) => {
if (!Number.isSafeInteger(nextCounter) || nextCounter < 0) {
throw new Error('Invalid seeded RNG counter');
}
counter = nextCounter;
},
enumerable: false
}
});
return rng;
}
/**
+339
View File
@@ -0,0 +1,339 @@
const { isDeepStrictEqual } = require('node:util');
const Game = require('./game');
const DungeonGenerator = require('./dungeon');
const SNAPSHOT_VERSION = 1;
const MAX_RNG_COUNTER = 100000000;
function invariant(condition, message) {
if (!condition) {
const error = new Error(`Invalid solo restart snapshot: ${message}`);
error.code = 'SOLO_RESTART_SNAPSHOT_INVALID';
throw error;
}
}
function integer(value, label, { min = Number.MIN_SAFE_INTEGER, max = Number.MAX_SAFE_INTEGER } = {}) {
invariant(Number.isSafeInteger(value), `${label} must be a safe integer`);
invariant(value >= min && value <= max, `${label} is outside the accepted range`);
return value;
}
function nullableInteger(value, label, bounds = {}) {
return value == null ? null : integer(value, label, bounds);
}
function finiteNumber(value, label, { min = -Number.MAX_VALUE, max = Number.MAX_VALUE } = {}) {
invariant(Number.isFinite(value), `${label} must be finite`);
invariant(value >= min && value <= max, `${label} is outside the accepted range`);
return value;
}
function nonemptyString(value, label, maxLength = 255) {
invariant(typeof value === 'string' && value.length > 0 && value.length <= maxLength,
`${label} must be a nonempty bounded string`);
return value;
}
function jsonClone(value, label) {
try {
const encoded = JSON.stringify(value);
invariant(typeof encoded === 'string', `${label} is not JSON serializable`);
return JSON.parse(encoded);
} catch (error) {
if (error?.code === 'SOLO_RESTART_SNAPSHOT_INVALID') throw error;
invariant(false, `${label} is not JSON serializable`);
}
}
function jsonValue(value, label) {
if (typeof value === 'string') {
try {
return JSON.parse(value);
} catch (_) {
invariant(false, `${label} is not valid JSON`);
}
}
return value;
}
function normalizedHex(value, label) {
const normalized = String(value || '').trim().toLowerCase();
invariant(/^[0-9a-f]{64}$/.test(normalized), `${label} must be 64 lowercase hex characters`);
return normalized;
}
function captureSoloRestartSnapshot(game, { paymentMonitoringActive = false } = {}) {
invariant(game && typeof game === 'object', 'game is required');
invariant(!game.settlementPending && !game.settlementCommitted,
'terminal or settlement-pending games cannot be captured');
invariant(game.gameState === 'active', 'only active games can be captured');
const dbId = integer(Number(game.dbId), 'game.dbId', { min: 1 });
const userId = integer(Number(game.dbUserId ?? game.userId), 'stable user id', { min: 1 });
const gameId = nonemptyString(game.id, 'game.id', 50);
const originalSocketId = nonemptyString(game.socketId, 'game.socketId');
invariant(typeof game.seededRNG?.getCounter === 'function', 'seeded RNG is not snapshot-capable');
const player = game.player?.getState?.();
const monster = game.monster;
invariant(player && monster, 'player and monster state are required');
const state = {
version: SNAPSHOT_VERSION,
gameId,
dbId,
userId,
proofTimestamp: integer(Number(game.gameProof?.timestamp), 'proof timestamp', { min: 0 }),
gameConfig: jsonClone(game.gameConfig, 'game config'),
difficultyConfig: jsonClone(game.difficultyConfig, 'difficulty config'),
depth: integer(Number(game.depth), 'depth', { min: 1 }),
maxDepth: integer(Number(game.maxDepth), 'max depth', { min: 1 }),
player: {
x: integer(Number(player.x), 'player.x'),
y: integer(Number(player.y), 'player.y'),
hasKey: player.hasKey === true,
hasTreasure: player.hasTreasure === true
},
monster: {
x: integer(Number(monster.x), 'monster.x'),
y: integer(Number(monster.y), 'monster.y'),
lastKnownPlayerX: nullableInteger(monster.lastKnownPlayerX, 'monster.lastKnownPlayerX'),
lastKnownPlayerY: nullableInteger(monster.lastKnownPlayerY, 'monster.lastKnownPlayerY'),
visionRange: finiteNumber(Number(monster.visionRange), 'monster.visionRange', { min: 1, max: 1000 })
},
moveCount: integer(Number(game.moveCount || 0), 'move count', { min: 0 }),
startedAt: integer(Number(game.startedAt), 'startedAt', { min: 0 }),
monsterMoveAccumulator: finiteNumber(Number(game.monsterMoveAccumulator || 0),
'monster move accumulator', { min: 0, max: 1000 }),
rngCounter: integer(Number(game.seededRNG.getCounter()), 'seeded RNG counter', {
min: 0,
max: MAX_RNG_COUNTER
}),
blockRec: nullableInteger(game.blockRec ?? game.blockHeight ?? null, 'entry block', { min: 0 }),
startBlock: nullableInteger(game.startBlock ?? null, 'start block', { min: 0 }),
fee: finiteNumber(Number(game.fee || 0), 'fee', { min: 0 })
};
invariant(state.depth <= state.maxDepth, 'depth exceeds maxDepth');
return {
gameId: dbId,
userId,
dungeonSeed: gameId,
snapshotVersion: SNAPSHOT_VERSION,
originalSocketId,
paymentMonitoringActive: paymentMonitoringActive === true,
state
};
}
function restoreSoloRestartSnapshot(row, { db } = {}) {
invariant(row && typeof row === 'object', 'database row is required');
invariant(db && typeof db.query === 'function', 'database handle is required');
invariant(row.status === 'active' && row.completed_at == null,
'database game must still be active and nonterminal');
const state = jsonValue(row.state, 'state');
invariant(state && typeof state === 'object' && !Array.isArray(state), 'state must be an object');
invariant(Number(row.snapshot_version) === SNAPSHOT_VERSION && state.version === SNAPSHOT_VERSION,
'unsupported snapshot version');
const dbId = integer(Number(row.game_id), 'game id', { min: 1 });
const userId = integer(Number(row.user_id), 'user id', { min: 1 });
const dungeonSeed = nonemptyString(String(row.dungeon_seed || ''), 'dungeon seed', 50);
const originalSocketId = nonemptyString(String(row.original_socket_id || ''), 'original socket id');
invariant(Number(row.joined_game_id ?? row.game_id) === dbId
&& Number(row.game_user_id ?? row.user_id) === userId
&& String(row.game_dungeon_seed ?? row.dungeon_seed) === dungeonSeed,
'snapshot identity does not exactly join its game row');
invariant(state.dbId === dbId && state.userId === userId && state.gameId === dungeonSeed,
'snapshot identity does not match its database anchors');
const snapshotCreatedAt = row.created_at instanceof Date
? row.created_at.getTime()
: Date.parse(row.created_at);
invariant(Number.isFinite(snapshotCreatedAt), 'snapshot created_at is invalid');
const now = Date.now();
invariant(snapshotCreatedAt <= now + 60000, 'snapshot created_at is in the future');
const serverSeed = normalizedHex(row.server_seed, 'server seed');
const commitment = normalizedHex(row.proof_commitment, 'proof commitment');
const effectiveSeed = normalizedHex(row.effective_seed, 'effective seed');
const clientSeed = String(row.client_seed ?? '').trim().toLowerCase();
invariant(clientSeed === '' || /^[0-9a-f]{1,64}$/.test(clientSeed), 'client seed is invalid');
let offerIssuedAt = null;
if (row.fairness_offer_issued_at != null) {
offerIssuedAt = row.fairness_offer_issued_at instanceof Date
? row.fairness_offer_issued_at.getTime()
: Number(row.fairness_offer_issued_at);
if (!Number.isFinite(offerIssuedAt)) offerIssuedAt = Date.parse(row.fairness_offer_issued_at);
invariant(Number.isSafeInteger(offerIssuedAt) && offerIssuedAt >= 0,
'fairness offer issued timestamp is invalid');
}
const gameConfig = jsonValue(state.gameConfig, 'game config');
const difficultyConfig = jsonValue(state.difficultyConfig, 'difficulty config');
invariant(gameConfig && typeof gameConfig === 'object' && !Array.isArray(gameConfig),
'game config must be an object');
invariant(difficultyConfig && typeof difficultyConfig === 'object' && !Array.isArray(difficultyConfig),
'difficulty config must be an object');
const persistedGeneratorVersion = String(row.generator_version || '').trim();
invariant(DungeonGenerator.SUPPORTED_GENERATOR_VERSIONS.includes(persistedGeneratorVersion),
'dungeon generator version is unsupported');
const persistedContext = jsonValue(row.proof_context, 'proof context');
const persistedFingerprints = jsonValue(row.layout_fingerprints, 'layout fingerprints');
invariant(persistedContext?.generatorVersion === persistedGeneratorVersion,
'proof context generator version does not match the game row');
const placeholderUser = {
id: originalSocketId,
joinGame() {},
endGame() {}
};
const game = Game.createRestoredStandardGame(originalSocketId, placeholderUser, {
...gameConfig,
fairnessProof: {
proofVersion: Number(row.proof_version) || 1,
offerId: row.fairness_offer_id || null,
offerIssuedAt,
serverSeed,
clientSeed,
commitment
}
}, persistedGeneratorVersion);
invariant(game.gameProof.seed === effectiveSeed, 'effective seed does not match the proof inputs');
invariant(game.gameProof.commitment === commitment, 'commitment does not match the server seed');
invariant(game.maxDepth === state.maxDepth, 'runtime maxDepth has drifted');
invariant(isDeepStrictEqual(jsonClone(game.gameConfig, 'restored game config'), gameConfig),
'runtime game configuration has drifted');
invariant(isDeepStrictEqual(jsonClone(game.difficultyConfig, 'restored difficulty config'), difficultyConfig),
'runtime difficulty configuration has drifted');
invariant(isDeepStrictEqual(game.gameProof.context, persistedContext), 'proof context has drifted');
invariant(isDeepStrictEqual(game.gameProof.layoutFingerprints, persistedFingerprints),
'layout fingerprint manifest has drifted');
invariant(game.gameProof.generatorVersion === persistedGeneratorVersion,
'dungeon generator version has drifted');
invariant(game.gameProof.layoutFingerprint === String(row.layout_fingerprint || '').trim(),
'level-one layout fingerprint has drifted');
const depth = integer(Number(state.depth), 'depth', { min: 1, max: state.maxDepth });
if (depth !== 1) game._generateLevel(depth);
game.depth = depth;
const width = integer(Number(game.width), 'game width', { min: 1 });
const height = integer(Number(game.height), 'game height', { min: 1 });
const player = state.player || {};
const monster = state.monster || {};
game.player.setState({
x: integer(Number(player.x), 'player.x', { min: 0, max: width - 1 }),
y: integer(Number(player.y), 'player.y', { min: 0, max: height - 1 }),
hasKey: player.hasKey === true,
hasTreasure: player.hasTreasure === true
});
game.monster.moveTo(
integer(Number(monster.x), 'monster.x', { min: 0, max: width - 1 }),
integer(Number(monster.y), 'monster.y', { min: 0, max: height - 1 })
);
game.monster.lastKnownPlayerX = nullableInteger(monster.lastKnownPlayerX,
'monster.lastKnownPlayerX', { min: 0, max: width - 1 });
game.monster.lastKnownPlayerY = nullableInteger(monster.lastKnownPlayerY,
'monster.lastKnownPlayerY', { min: 0, max: height - 1 });
game.monster.visionRange = finiteNumber(Number(monster.visionRange),
'monster.visionRange', { min: 1, max: 1000 });
const primaryFloor = game.gameConfig.primaryFloor || "'1";
const secondaryFloor = game.gameConfig.secondaryFloor || "'2";
const passable = (x, y) => {
const tile = game.dungeon?.map?.[y]?.[x];
return tile === primaryFloor || tile === secondaryFloor
|| tile === 0 || tile === '>' || tile === '$M';
};
invariant(passable(game.player.x, game.player.y), 'player is not on a passable tile');
invariant(passable(game.monster.x, game.monster.y), 'monster is not on a passable tile');
invariant(game.player.x !== game.monster.x || game.player.y !== game.monster.y,
'active player and monster overlap');
const onExit = Array.isArray(game.dungeon?.exit)
&& game.player.x === game.dungeon.exit[0]
&& game.player.y === game.dungeon.exit[1];
invariant(!onExit, 'active player is already on the exit');
const generatedTreasure = Array.isArray(game.dungeon?.treasure)
? [...game.dungeon.treasure]
: null;
if (game.player.hasTreasure) {
invariant(depth === state.maxDepth, 'treasure cannot be held before the final depth');
invariant(generatedTreasure !== null, 'final-depth treasure cannot be reconstructed');
// Picking treasure removes the live dungeon marker immediately. The player may still
// stand on the former pickup coordinate; it is no longer an active treasure tile.
game.dungeon.treasure = null;
} else if (depth === state.maxDepth) {
invariant(generatedTreasure !== null, 'uncollected final-depth treasure is missing');
const onUncollectedTreasure = game.player.x === generatedTreasure[0]
&& game.player.y === generatedTreasure[1];
invariant(!onUncollectedTreasure, 'active player is already on uncollected treasure');
} else {
invariant(generatedTreasure === null, 'treasure exists before the final depth');
}
game.moveCount = integer(Number(state.moveCount), 'move count', { min: 0 });
game.startedAt = integer(Number(state.startedAt), 'startedAt', { min: 0 });
invariant(game.startedAt <= snapshotCreatedAt + 60000 && game.startedAt <= now + 60000,
'startedAt is later than the durable snapshot');
game.monsterMoveAccumulator = finiteNumber(Number(state.monsterMoveAccumulator),
'monster move accumulator', { min: 0, max: 1000 });
const generatedRngCounter = game.seededRNG.getCounter();
const restoredRngCounter = integer(Number(state.rngCounter), 'seeded RNG counter', {
min: 0,
max: MAX_RNG_COUNTER
});
invariant(restoredRngCounter >= generatedRngCounter,
'seeded RNG counter predates deterministic dungeon generation');
game.seededRNG.setCounter(restoredRngCounter);
game.blockRec = nullableInteger(state.blockRec, 'entry block', { min: 0 });
const durableEntryBlock = nullableInteger(
row.start_block_height == null ? null : Number(row.start_block_height),
'durable entry block',
{ min: 0 }
);
invariant(game.blockRec === durableEntryBlock,
'entry block does not match the durable game row');
game.blockHeight = game.blockRec;
game.startBlock = nullableInteger(state.startBlock, 'start block', { min: 0 });
game.fee = finiteNumber(Number(state.fee), 'fee', { min: 0 });
game.gameState = 'active';
game.updateFOV();
game.id = dungeonSeed;
game.socketId = originalSocketId;
game.dbId = dbId;
game.userId = userId;
game.dbUserId = userId;
game.db = db;
game.gameMode = String(row.game_mode || 'FREE').toUpperCase();
game.gameProof.gameId = dungeonSeed;
game.gameProof.timestamp = integer(Number(state.proofTimestamp), 'proof timestamp', { min: 0 });
invariant(game.gameProof.timestamp <= snapshotCreatedAt + 60000
&& game.gameProof.timestamp >= game.startedAt - 60000,
'proof timestamp is outside the game/snapshot lifetime');
game.gameProof.proofVersion = Number(row.proof_version) || 1;
game.gameProof.offerId = row.fairness_offer_id || null;
game.gameProof.offerIssuedAt = offerIssuedAt;
game.gameProof.context = persistedContext;
game.gameProof.layoutFingerprints = persistedFingerprints;
game.gameProof.layoutFingerprint = String(row.layout_fingerprint || '').trim();
return {
game,
userId,
originalSocketId,
paymentMonitoringActive: row.payment_monitoring_active === true,
snapshotVersion: SNAPSHOT_VERSION
};
}
module.exports = {
SNAPSHOT_VERSION,
captureSoloRestartSnapshot,
restoreSoloRestartSnapshot
};
+39 -2
View File
@@ -1137,6 +1137,13 @@ async function startServer() {
if (!sm || typeof sm.recoverOrphanedGames !== 'function') {
throw new Error('Orphaned solo-game financial recovery is unavailable.');
}
// A graceful predecessor may have frozen exact nonterminal solo runtime state. Rebuild
// those games into the reconnect cache before orphan recovery scans active rows; invalid
// or drifted snapshots abort startup before a listener or financial dispatcher exists.
const restartRestore = await socketHandlers.rehydrateSoloRestartSnapshots();
if (restartRestore.restored > 0) {
console.log(`🔄 Rehydrated ${restartRestore.restored} durable solo restart snapshot(s)`);
}
await sm.recoverOrphanedGames();
// Match queue recovery and payout-liability reconciliation require the migrations that
@@ -1310,6 +1317,13 @@ async function gracefulShutdown(signal, exitCode = 0) {
if (databaseHealthInterval) clearInterval(databaseHealthInterval);
financialEventExporter.stop();
// Work that crossed the synchronous shutdown gate may still be completing a game start or a
// sequential block-timeout pass. Drain every such producer before GameManager snapshots its
// terminal settlement set; otherwise a paused timeout pass could add a later result behind it.
if (socketHandlers?.drainShutdownProducers) {
await socketHandlers.drainShutdownProducers();
}
// Terminal solo games are held in a serialized settlement-pending state when their atomic
// completion transaction fails. Drain those retries before disabling GameModeManager or
// closing PostgreSQL so a normal deploy/restart cannot turn a known result into an orphan.
@@ -1317,10 +1331,27 @@ async function gracefulShutdown(signal, exitCode = 0) {
const soloDrain = await socketHandlers.gameManager.shutdown({ timeoutMs: 4000 });
if (soloDrain.pending > 0) {
console.error(` ${soloDrain.pending} solo settlement(s) remain unresolved after shutdown drain`);
const error = new Error('Refusing graceful restart with unresolved solo settlements');
error.code = 'SOLO_SETTLEMENT_DRAIN_INCOMPLETE';
throw error;
} else if (soloDrain.initial > 0) {
console.log(` Settled ${soloDrain.settled} terminal solo game(s) during shutdown drain`);
}
}
// Only controlled termination signals may serialize live runtime objects. After an uncaught
// exception/rejection process state is not trustworthy; startup's orphan recovery remains the
// explicit economic fallback for those crash paths.
if (signal === 'SIGTERM' || signal === 'SIGINT') {
const restartSnapshots = await socketHandlers.persistSoloRestartSnapshots();
if (restartSnapshots.captured > 0) {
console.log(` Persisted ${restartSnapshots.captured} active solo game(s) for restart`);
}
}
// Snapshot ownership is now durable. Close Socket.IO transports so no existing connection
// survives into DB teardown; disconnect cleanup may adjust in-memory maps but cannot erase the
// PostgreSQL restart snapshots.
await new Promise((resolve) => io.close(() => resolve()));
gameModeManager.shutdown();
if (payoutRetryService) payoutRetryService.stop();
if (alertService) alertService.stopPeriodicChecks?.();
@@ -1373,5 +1404,11 @@ process.on('unhandledRejection', (reason) => {
gracefulShutdown('unhandledRejection', 1).catch(() => process.exit(1));
});
process.on('SIGTERM', () => gracefulShutdown('SIGTERM', 0));
process.on('SIGINT', () => gracefulShutdown('SIGINT', 0));
process.on('SIGTERM', () => gracefulShutdown('SIGTERM', 0).catch((error) => {
console.error('Graceful SIGTERM failed:', error?.message || error);
process.exit(1);
}));
process.on('SIGINT', () => gracefulShutdown('SIGINT', 0).catch((error) => {
console.error('Graceful SIGINT failed:', error?.message || error);
process.exit(1);
}));
@@ -0,0 +1,43 @@
-- Durable, versioned snapshots for nonterminal solo games during a graceful process restart.
--
-- The existing suspended-game manager is intentionally an in-memory reconnect cache. A systemd
-- restart destroys that cache, so a controlled shutdown must first anchor the exact runtime state
-- to the durable game/user identity. Snapshots never authorize a payout or alter accepted economic
-- terms; they only allow the same active run to be reconstructed before public admission resumes.
CREATE TABLE IF NOT EXISTS solo_restart_snapshots (
game_id INTEGER PRIMARY KEY REFERENCES games(id) ON DELETE CASCADE,
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
dungeon_seed VARCHAR(50) NOT NULL,
snapshot_version SMALLINT NOT NULL,
original_socket_id VARCHAR(255) NOT NULL,
payment_monitoring_active BOOLEAN NOT NULL DEFAULT FALSE,
state JSONB NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
CONSTRAINT solo_restart_snapshot_version_check CHECK (snapshot_version = 1),
CONSTRAINT solo_restart_snapshot_state_object_check CHECK (jsonb_typeof(state) = 'object'),
CONSTRAINT solo_restart_snapshot_user_once UNIQUE (user_id),
CONSTRAINT solo_restart_snapshot_game_seed_once UNIQUE (dungeon_seed)
);
CREATE INDEX IF NOT EXISTS idx_solo_restart_snapshots_created
ON solo_restart_snapshots (created_at);
-- A terminal game can never retain a resumable state. Keeping this invariant in PostgreSQL makes
-- every completion path (including future/manual paths) clean up the snapshot automatically.
CREATE OR REPLACE FUNCTION delete_terminal_solo_restart_snapshot()
RETURNS TRIGGER AS $$
BEGIN
IF NEW.status IS DISTINCT FROM 'active' THEN
DELETE FROM solo_restart_snapshots WHERE game_id = NEW.id;
END IF;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS trg_games_delete_terminal_restart_snapshot ON games;
CREATE TRIGGER trg_games_delete_terminal_restart_snapshot
AFTER UPDATE OF status ON games
FOR EACH ROW
WHEN (NEW.status IS DISTINCT FROM 'active')
EXECUTE FUNCTION delete_terminal_solo_restart_snapshot();
+60 -2
View File
@@ -47,6 +47,9 @@ class PaymentHandlers {
this._confirmedRetentionMs = 6 * 60 * 60 * 1000; // 6 hours
// Track expiry timeouts so they can be cleared / unref'd to avoid keeping process open
this._expiryTimeouts = new Map(); // socketId -> timeout
this._isShuttingDown = false;
this._statusUpdatesInFlight = new Set();
this._invoiceCreationsInFlight = new Set();
// Keep reference so tests can dispose / and unref so it doesn't keep event loop alive
this._cleanupInterval = setInterval(() => {
const now = Date.now();
@@ -208,7 +211,29 @@ class PaymentHandlers {
}
}
_runInvoiceCreation(socket, task) {
if (this._isShuttingDown) {
this.io?.to?.(socket.id)?.emit?.('payment_error', {
error: 'The server is restarting; new payment requests are temporarily closed.',
code: 'SERVER_SHUTTING_DOWN'
});
return Promise.resolve(null);
}
let work;
try {
work = Promise.resolve(task());
} catch (error) {
work = Promise.reject(error);
}
this._invoiceCreationsInFlight.add(work);
return work.finally(() => this._invoiceCreationsInFlight.delete(work));
}
async handlePaymentRequest(socket, data) {
return this._runInvoiceCreation(socket, () => this._handlePaymentRequestActive(socket, data));
}
async _handlePaymentRequestActive(socket, data) {
if (!this.gameModeManager) {
this.clearPendingCommerceAcknowledgement(socket.id);
this.io.to(socket.id).emit('payment_error', { error: 'Payment system not available' });
@@ -325,6 +350,11 @@ class PaymentHandlers {
}
async createAndShowPaymentRequest(socket, options = {}) {
return this._runInvoiceCreation(socket,
() => this._createAndShowPaymentRequestActive(socket, options));
}
async _createAndShowPaymentRequestActive(socket, options = {}) {
if (!this.gameModeManager) {
this.clearPendingCommerceAcknowledgement(socket.id);
this.pendingEntryFairness.delete(socket.id);
@@ -559,10 +589,37 @@ class PaymentHandlers {
}
}
/** Stop new monitor callbacks from creating queue/game/entitlement work. */
beginShutdown() {
this._isShuttingDown = true;
}
async drainShutdownWork() {
while (this._statusUpdatesInFlight.size > 0 || this._invoiceCreationsInFlight.size > 0) {
await Promise.allSettled([
...this._statusUpdatesInFlight,
...this._invoiceCreationsInFlight
]);
}
return { pending: 0 };
}
// Handle one payment status update (raw wallet-style shape: in_mempool/confirmed/complete/
// amount/required/confirmations). Extracted verbatim from the monitor callback so native and
// gateway providers share the exact same confirmation logic.
// amount/required/confirmations). A callback that began before the synchronous shutdown gate
// is tracked to completion; later callbacks do no admission work and startup recovery observes
// the underlying durable payment state.
async _handlePaymentStatus(socket, paymentRequest, currentUser, status) {
if (this._isShuttingDown) return;
const update = this._handlePaymentStatusActive(socket, paymentRequest, currentUser, status);
this._statusUpdatesInFlight.add(update);
try {
return await update;
} finally {
this._statusUpdatesInFlight.delete(update);
}
}
async _handlePaymentStatusActive(socket, paymentRequest, currentUser, status) {
if (status.in_mempool && !status.confirmed) {
if (this.mempoolNotified.has(paymentRequest.address)) return;
this.mempoolNotified.set(paymentRequest.address, Date.now());
@@ -897,6 +954,7 @@ class PaymentHandlers {
* Dispose resources (intervals, monitors) - useful for tests / graceful shutdown.
*/
dispose() {
this.beginShutdown();
if (this._cleanupInterval) {
clearInterval(this._cleanupInterval);
this._cleanupInterval = null;
+15 -4
View File
@@ -460,7 +460,13 @@ class SessionManager {
let orphans;
try {
orphans = await this.db.query(
`SELECT id FROM games WHERE status = 'active' ORDER BY id ASC`,
`SELECT g.id
FROM games g
WHERE g.status = 'active'
AND NOT EXISTS (
SELECT 1 FROM solo_restart_snapshots s WHERE s.game_id = g.id
)
ORDER BY g.id ASC`,
[]
);
} catch (error) {
@@ -480,9 +486,14 @@ class SessionManager {
// Lock the game row and re-check under the lock: another instance (or a prior
// run) may already have finalized it.
const lock = await client.query(
`SELECT id, user_id, game_mode, payment_id, status,
entry_consumed_at, entry_credits_spent
FROM games WHERE id = $1 FOR UPDATE`,
`SELECT g.id, g.user_id, g.game_mode, g.payment_id, g.status,
g.entry_consumed_at, g.entry_credits_spent
FROM games g
WHERE g.id = $1
AND NOT EXISTS (
SELECT 1 FROM solo_restart_snapshots s WHERE s.game_id = g.id
)
FOR UPDATE`,
[orphan.id]
);
const game = lock.rows[0];
+328 -60
View File
@@ -25,6 +25,11 @@ const IdentityService = require('./identityService');
const SuspendedGameManager = require('./suspendedGameManager');
const MemoryManager = require('../utils/memoryManager');
const FairnessOfferManager = require('../game/fairnessOfferManager');
const {
SNAPSHOT_VERSION: SOLO_RESTART_SNAPSHOT_VERSION,
captureSoloRestartSnapshot,
restoreSoloRestartSnapshot
} = require('../game/soloRestartSnapshot');
const { normalizeError } = require('../utils/errors');
const money = require('../money/atomic');
const {
@@ -62,6 +67,7 @@ class SocketHandlers {
this.MOVE_COOLDOWN = 100; // Minimum 100ms between moves
this.fairnessOffers = new FairnessOfferManager();
this._isShuttingDown = false;
this._admissionsInFlight = new Set();
// Initialize rate limiter with debug mode matching debugManager
this.rateLimiter = new RateLimiter({
@@ -320,10 +326,194 @@ class SocketHandlers {
/** Stop new solo movement before taking the graceful-shutdown settlement snapshot. */
beginShutdown() {
// This flag is the synchronous boundary: every admission wrapper and lower-level game
// creator observes it before any shutdown await can yield back to a client callback.
this._isShuttingDown = true;
this.gameManager?.beginShutdown?.();
this.gameModeManager?.beginGameAdmissionShutdown?.();
this.paymentHandlers?.beginShutdown?.();
this.movementManager?.shutdown?.();
}
_emitShutdownAdmissionRefusal(socket, kind = 'admission') {
const payload = {
code: 'SERVER_SHUTTING_DOWN',
message: 'The server is restarting; new entries are temporarily closed.'
};
try { socket?.emit?.('server_restarting', { ...payload, kind }); } catch (_) {}
try { this.broadcastManager?.sendStatusUpdate?.(socket?.id, 'warning', payload.message); } catch (_) {}
return payload;
}
_rejectAdmissionDuringShutdown(socket, kind) {
if (!this._isShuttingDown) return false;
this._emitShutdownAdmissionRefusal(socket, kind);
return true;
}
_runAdmission(socket, kind, task) {
if (this._rejectAdmissionDuringShutdown(socket, kind)) return Promise.resolve(null);
const token = {};
this._admissionsInFlight.add(token);
let work;
try {
work = Promise.resolve(task());
} catch (error) {
work = Promise.reject(error);
}
token.work = work;
return work.finally(() => this._admissionsInFlight.delete(token));
}
async drainAdmissionHandlers() {
while (this._admissionsInFlight.size > 0) {
await Promise.allSettled(Array.from(this._admissionsInFlight, token => token.work));
}
return { pending: 0 };
}
/**
* Drain every operation that could still create a game or install a terminal solo result.
* This must finish before GameManager.shutdown() snapshots/retries its settlement map: an
* already-running block timeout pass is itself an admission producer and can otherwise resume
* after an await and add a new settlement behind the drain.
*/
async drainShutdownProducers() {
await this.drainAdmissionHandlers();
await this.paymentHandlers?.drainShutdownWork?.();
await this.gameModeManager?.drainGameStartAdmissions?.();
await this.gameManager?.drainAdmissions?.();
return { pending: 0 };
}
/**
* Persist every nonterminal solo run that currently exists only as a live Game object.
* Movement is already frozen by beginShutdown(), and the caller drains terminal settlement
* intents before invoking this method. One transaction prevents a partial restart boundary:
* either every eligible run has a guarded snapshot or none of the new snapshots commit.
*/
async persistSoloRestartSnapshots() {
const db = this.gameModeManager?.db;
if (!db || typeof db.withTransaction !== 'function') {
throw new Error('Solo restart snapshots require a transactional database handle');
}
// Work that crossed the synchronous gate is allowed to reach a stable DB/in-memory state;
// nothing new can enter behind it. Only then enumerate the complete snapshot set.
await this.drainShutdownProducers();
const captures = new Map();
const addGame = (game, socketId, paymentMonitoringActive) => {
if (!game) return;
if (game.settlementPending || game.settlementCommitted || game.gameState === 'ended') {
throw new Error(`Refusing to snapshot terminal solo game ${game.id || 'unknown'}`);
}
// A snapshot rehydrated earlier in this same process remains durable until its owner
// claims it; it does not need (and must not receive) a second write on shutdown.
const captured = captureSoloRestartSnapshot(game, { paymentMonitoringActive });
const previous = captures.get(captured.gameId);
if (previous && previous.dungeonSeed !== captured.dungeonSeed) {
throw new Error(`Conflicting restart snapshot identity for game row ${captured.gameId}`);
}
captures.set(captured.gameId, captured);
};
for (const [socketId, game] of this.activeGames.entries()) {
addGame(game, socketId, this.paymentHandlers?.hasActiveMonitoring?.(socketId) === true);
}
for (const suspended of this.suspendedGameManager?.suspendedGames?.values?.() || []) {
if (suspended?.durableRestartSnapshot === true) continue;
addGame(suspended?.game, suspended?.originalSocketId,
suspended?.paymentMonitoringActive === true);
}
if (captures.size === 0) return { captured: 0 };
await db.withTransaction(async (client) => {
for (const snapshot of captures.values()) {
const result = await client.query(`
INSERT INTO solo_restart_snapshots (
game_id, user_id, dungeon_seed, snapshot_version, original_socket_id,
payment_monitoring_active, state, created_at
)
SELECT g.id, g.user_id, g.dungeon_seed, $4, $5, $6, $7::jsonb, NOW()
FROM games g
WHERE g.id = $1 AND g.user_id = $2 AND g.dungeon_seed = $3
AND g.status = 'active' AND g.completed_at IS NULL
ON CONFLICT (game_id) DO UPDATE SET
snapshot_version = EXCLUDED.snapshot_version,
original_socket_id = EXCLUDED.original_socket_id,
payment_monitoring_active = EXCLUDED.payment_monitoring_active,
state = EXCLUDED.state,
created_at = EXCLUDED.created_at
WHERE solo_restart_snapshots.user_id = EXCLUDED.user_id
AND solo_restart_snapshots.dungeon_seed = EXCLUDED.dungeon_seed
RETURNING game_id
`, [
snapshot.gameId,
snapshot.userId,
snapshot.dungeonSeed,
snapshot.snapshotVersion,
snapshot.originalSocketId,
snapshot.paymentMonitoringActive,
JSON.stringify(snapshot.state)
]);
if (result.rowCount !== 1) {
throw new Error(`Active solo game ${snapshot.gameId} changed during restart snapshot`);
}
}
});
return { captured: captures.size };
}
/** Reconstruct durable graceful-restart snapshots before orphan recovery or admission. */
async rehydrateSoloRestartSnapshots() {
const db = this.gameModeManager?.db;
if (!db || typeof db.query !== 'function') {
throw new Error('Solo restart rehydration requires a database handle');
}
const result = await db.query(`
SELECT s.game_id, s.user_id, s.dungeon_seed, s.snapshot_version,
s.original_socket_id, s.payment_monitoring_active, s.state, s.created_at,
g.id AS joined_game_id, g.user_id AS game_user_id,
g.dungeon_seed AS game_dungeon_seed,
g.status, g.completed_at, g.game_mode, g.start_block_height,
g.proof_version, g.fairness_offer_id, g.fairness_offer_issued_at,
g.proof_commitment, g.server_seed, g.client_seed, g.effective_seed,
g.layout_fingerprint, g.layout_fingerprints, g.generator_version,
g.proof_context
FROM solo_restart_snapshots s
LEFT JOIN games g
ON g.id = s.game_id
AND g.user_id = s.user_id
AND g.dungeon_seed = s.dungeon_seed
ORDER BY s.game_id ASC
`, []);
let restored = 0;
for (const row of result.rows || []) {
const state = restoreSoloRestartSnapshot(row, { db });
if (this.suspendedGameManager.hasSuspendedGame(state.userId)) {
throw new Error(`Duplicate suspended solo identity ${state.userId}`);
}
const accepted = this.suspendedGameManager.suspendGame(
state.userId,
state.originalSocketId,
state.game,
{
paymentMonitoringActive: state.paymentMonitoringActive,
durableRestartSnapshot: true,
snapshotVersion: SOLO_RESTART_SNAPSHOT_VERSION,
blockRec: state.game.blockRec
}
);
if (!accepted) throw new Error(`Failed to rehydrate solo game ${state.game.id}`);
restored += 1;
}
return { restored };
}
/** Dynamic financial-recovery gate used by readiness and paid admission. */
isFinancialRecoveryReady() {
return this.matchQueue?.initialized === true
@@ -432,8 +622,20 @@ class SocketHandlers {
* Initialize socket event handlers for a new connection
*/
async handleConnection(socket) {
if (this._isShuttingDown) {
this._emitShutdownAdmissionRefusal(socket, 'connection');
socket.disconnect(true);
return;
}
const connectionResult = await this.connectionHandler.handleConnection(socket);
if (!connectionResult) return; // Connection was rejected or failed
// A connection may have entered just before beginShutdown() and awaited its DB session.
// Recheck before restoring state or registering any mutating event handlers.
if (this._isShuttingDown) {
this._emitShutdownAdmissionRefusal(socket, 'connection');
socket.disconnect(true);
return;
}
// Publish the server commitment before the client can submit a client seed. register_client
// re-emits this same offer (never swaps it) in case the browser attached listeners late.
@@ -447,7 +649,22 @@ class SocketHandlers {
if (sessionInfo && sessionInfo.resumed && dbUserId && this.suspendedGameManager) {
if (this.suspendedGameManager.hasSuspendedGame(dbUserId)) {
const memUser = this.connectionHandler.getUserBySocket(socket.id);
const restored = this.suspendedGameManager.restoreGame(dbUserId, socket.id, memUser);
let restored;
try {
restored = await this.suspendedGameManager.restoreGame(dbUserId, socket.id, memUser);
} catch (error) {
// Never let a failed durable claim fall through to queue/new-game admission:
// the database still owns an active run for this identity. A reconnect can
// retry after the transient fault or an operator can resolve the snapshot.
console.error(`[handleConnection] Suspended game restore failed for user ${dbUserId}:`,
normalizeError(error).message);
socket.emit('session_restore_failed', {
code: 'ACTIVE_GAME_RESTORE_FAILED',
message: 'Your active game could not be restored safely. Please reconnect shortly.'
});
socket.disconnect(true);
return;
}
if (restored) {
restoredGame = restored.game;
@@ -549,11 +766,14 @@ class SocketHandlers {
socket.emit('active_games', gameList);
}
// Register event handlers
// Register event handlers. Admission wrappers remain live for the life of this socket and
// consult the dynamic shutdown flag, so existing connections cannot race the snapshot.
const runAdmission = (kind, task) => this._runAdmission(socket, kind, task);
socket.on('chat message', (msg) => this.chatHandler.handleChatMessage(socket, msg, {
// Legacy text command has no client contribution, but still consumes the already
// published one-time server offer with an empty clientSeed.
handleGameQueue: (socket) => this.handleJoinQueue(socket, {}),
handleGameQueue: (commandSocket) => runAdmission('chat_game_queue',
() => this.handleJoinQueue(commandSocket, {})),
handleCancelEntry: (socket) => this.queueHandler.handleCancelEntry(socket),
handleStatsRequest: (socket) => this.handleStatsRequest(socket)
}));
@@ -586,15 +806,20 @@ class SocketHandlers {
if (this.debugManager?.CONSOLE_LOGGING) console.warn('Failed to re-send identity snapshot:', err.message);
});
});
socket.on('auto_start', (data) => this.handleAutoStart(socket, (data && typeof data === 'object') ? data : {})); // New handler for start button
socket.on('play_free', (data) => this.handleAutoStart(socket, { ...((data && typeof data === 'object') ? data : {}), free: true })); // Explicit free-play choice
socket.on('join_queue', (data) => this.handleJoinQueue(socket, (data && typeof data === 'object') ? data : {})); // Queue instead of auto-start ({ free: true } for Pleb-board free play)
socket.on('early_entry', (data) => this.handleEarlyEntry(socket, (data && typeof data === 'object') ? data : {})); // Early entry without waiting for block
socket.on('auto_start', (data) => runAdmission('auto_start',
() => this.handleAutoStart(socket, (data && typeof data === 'object') ? data : {}))); // New handler for start button
socket.on('play_free', (data) => runAdmission('play_free',
() => this.handleAutoStart(socket, { ...((data && typeof data === 'object') ? data : {}), free: true }))); // Explicit free-play choice
socket.on('join_queue', (data) => runAdmission('join_queue',
() => this.handleJoinQueue(socket, (data && typeof data === 'object') ? data : {}))); // Queue instead of auto-start ({ free: true } for Pleb-board free play)
socket.on('early_entry', (data) => runAdmission('early_entry',
() => this.handleEarlyEntry(socket, (data && typeof data === 'object') ? data : {}))); // Early entry without waiting for block
socket.on('fairness_offer_request', () => this._emitFairnessOffer(socket));
socket.on('address:prompt', () => this.handleAddressPrompt(socket));
// Payment system handlers
socket.on('request_payment', (data) => this.handlePaymentRequest(socket, data));
socket.on('request_payment', (data) => runAdmission('request_payment',
() => this.handlePaymentRequest(socket, data)));
socket.on('check_payment_status', (data) => this.handleCheckPaymentStatus(socket, data));
socket.on('get_user_credits', () => this.handleGetUserCredits(socket));
socket.on('address:update', (data) => this.handleAddressUpdate(socket, data));
@@ -620,7 +845,8 @@ class SocketHandlers {
// never crash the connection or leak a stack to the client — it logs and emits a
// benign 'match_error' instead (C5). Identity is always resolved from the CONNECTION,
// never from the client payload.
socket.on('match_queue', (data) => this._handleMatchQueue(socket, data));
socket.on('match_queue', (data) => runAdmission('match_queue',
() => this._handleMatchQueue(socket, data)));
socket.on('match_move', (data) => {
try {
this.matchManager.move(socket, data);
@@ -648,6 +874,10 @@ class SocketHandlers {
}
async handlePaymentRequest(socket, data = {}) {
if (this._isShuttingDown === true) {
this._rejectAdmissionDuringShutdown?.(socket, 'request_payment');
return;
}
const request = (data && typeof data === 'object') ? data : {};
const rlId = stableId(socket, this.sessionManager);
const rlIp = clientIp(socket);
@@ -695,6 +925,10 @@ class SocketHandlers {
* If eligible, starts game immediately (bypassing block queue) and processes game start (credit deduction / payment linkage)
*/
async handleAutoStart(socket, opts = {}) {
if (this._isShuttingDown === true) {
this._rejectAdmissionDuringShutdown?.(socket, 'solo_start');
return;
}
try {
const wantsExplicitFree = opts.free === true && this.gameModeManager?.freePlayEnabled;
if (!wantsExplicitFree
@@ -843,6 +1077,10 @@ class SocketHandlers {
* Handle join_queue request — adds player to block queue instead of starting immediately
*/
async handleJoinQueue(socket, opts = {}) {
if (this._isShuttingDown === true) {
this._rejectAdmissionDuringShutdown?.(socket, 'solo_queue');
return;
}
try {
const rlId = stableId(socket, this.sessionManager);
const rlIp = clientIp(socket);
@@ -890,6 +1128,10 @@ class SocketHandlers {
* Only available for free mode and credits mode (not direct payment mode)
*/
async handleEarlyEntry(socket, opts = {}) {
if (this._isShuttingDown === true) {
this._rejectAdmissionDuringShutdown?.(socket, 'early_entry');
return;
}
try {
// Early entry is free only on a free-only instance. Mixed/credits entry may consume a
// credit, so treat ambiguous mixed-mode requests as paid and fail closed.
@@ -1013,6 +1255,10 @@ class SocketHandlers {
}
const action = (data && data.action === 'leave') ? 'leave' : 'join';
if (action === 'join' && this._isShuttingDown === true) {
this._rejectAdmissionDuringShutdown?.(socket, 'match_queue');
return;
}
if (action === 'join' && this.activeGames?.has(socket.id)) {
const soloGame = this.activeGames.get(socket.id);
socket.emit('match_error', {
@@ -1503,68 +1749,85 @@ class SocketHandlers {
* Start games for waiting players when a new block is found
*/
async startGamesForWaiting(blockHeight) {
// Start queued single-player games first and capture the result to return.
const result = await this.queueHandler.startGamesForWaiting(blockHeight);
// Then drain match-mode queues on the same block, fire-and-forget so a match-layer
// error can never block or fail single-player block processing (MP-C3).
if (this.matchScheduler) {
this.matchScheduler.onBlock(blockHeight).catch(err => {
if (this.debugManager.CONSOLE_LOGGING) console.error('[SocketHandlers] Match scheduler block error:', err);
});
}
return result;
if (this._isShuttingDown) return null;
return this._runAdmission(null, 'block_queue_start', async () => {
// Start queued single-player games first and capture the result to return.
const result = await this.queueHandler.startGamesForWaiting(blockHeight);
// Keep scheduler work inside the tracked admission boundary. A match-layer fault still
// cannot fail solo processing, but shutdown now knows when the pre-barrier block tick
// has reached a stable state.
if (this.matchScheduler) {
await this.matchScheduler.onBlock(blockHeight).catch(err => {
if (this.debugManager.CONSOLE_LOGGING) console.error('[SocketHandlers] Match scheduler block error:', err);
});
}
return result;
});
}
/**
* Check for game timeouts based on block height
*/
async checkGamesTimeout(currentHeight) {
// Snapshot the entries first: handleGameOver mutates activeGames (it deletes the
// entry), so iterating the live Map could skip games. Process sequentially and
// await each game-over rather than firing them all off concurrently on one tick.
// Anti-instant-death guard ONLY (default 2s, set 0 to disable). Random block timing
// is the game's core mechanic and is deliberately preserved — this is not a fairness
// floor. It only avoids the degenerate case where a block lands in the same instant
// the game starts and the player dies before the dungeon even renders / before their
// first move is possible (100ms move cooldown).
const graceMs = (() => { const v = parseInt(process.env.GAME_START_GRACE_MS, 10); return Number.isFinite(v) ? v : 2000; })();
const now = Date.now();
const snapshot = Array.from(this.activeGames.entries());
for (const [socketId, game] of snapshot) {
// G3: the snapshot can go stale while we await a previous game-over. If this game
// was concurrently won or otherwise ended (removed from activeGames on another
// async path), never re-end it here as a loss/timeout.
if (!this.activeGames.has(socketId)) continue;
// Terminal games deliberately remain mapped until their DB transaction succeeds.
// Never race that frozen result with a later block-timeout loss.
if (game?.settlementPending || game?.settlementCommitted || game?.gameState === 'ended') continue;
if (this._isShuttingDown) return;
return this._runAdmission(null, 'block_game_timeout', async () => {
// Snapshot the entries first: handleGameOver mutates activeGames (it deletes the
// entry), so iterating the live Map could skip games. Process sequentially and
// await each game-over rather than firing them all off concurrently on one tick.
// Anti-instant-death guard ONLY (default 2s, set 0 to disable). Random block timing
// is the game's core mechanic and is deliberately preserved — this is not a fairness
// floor. It only avoids the degenerate case where a block lands in the same instant
// the game starts and the player dies before the dungeon even renders / before their
// first move is possible (100ms move cooldown).
const graceMs = (() => {
const value = parseInt(process.env.GAME_START_GRACE_MS, 10);
return Number.isFinite(value) ? value : 2000;
})();
const now = Date.now();
const snapshot = Array.from(this.activeGames.entries());
for (const [socketId, game] of snapshot) {
// A pass can have crossed the admission gate before shutdown and then yielded
// while settling an earlier game. Never let it install a later terminal intent
// after the shutdown producer drain has begun.
if (this._isShuttingDown) break;
// G3: the snapshot can go stale while we await a previous game-over. If this game
// was concurrently won or otherwise ended (removed from activeGames on another
// async path), never re-end it here as a loss/timeout.
if (!this.activeGames.has(socketId)) continue;
// Terminal games deliberately remain mapped until their DB transaction succeeds.
// Never race that frozen result with a later block-timeout loss.
if (game?.settlementPending || game?.settlementCommitted || game?.gameState === 'ended') continue;
const user = this.connectionHandler.getUserBySocket(socketId);
if (!user) continue;
const user = this.connectionHandler.getUserBySocket(socketId);
if (!user) continue;
// G1/C3: only end when the entry block is DEFINED and a later block has arrived.
// If we've never recorded an entry block for this player, record the current
// height as the first observation instead of treating a missing value as either
// immortal (never times out) or instant-death (times out on the first tick).
if (user.blockRec == null) {
user.blockRec = currentHeight;
continue;
}
const elapsedMs = now - (game.startedAt || now);
if (currentHeight > user.blockRec && elapsedMs >= graceMs) {
if (this.debugManager.CONSOLE_LOGGING) {
console.log(`💀 GAME TIMEOUT for player ${socketId}: entered on block ${user.blockRec}, died on block ${currentHeight}`);
// G1/C3: only end when the entry block is DEFINED and a later block has arrived.
// If we've never recorded an entry block for this player, record the current
// height as the first observation instead of treating a missing value as either
// immortal (never times out) or instant-death (times out on the first tick).
if (user.blockRec == null) {
user.blockRec = currentHeight;
continue;
}
game.gameState = 'lost';
const fakeSocket = { id: socketId };
try {
await this.handleGameOver(fakeSocket, game, 'lost', 'timeout', 'You didn\'t escape before the block time limit!');
} catch (e) {
console.error(`checkGamesTimeout: handleGameOver failed for ${socketId}:`, e.message);
const elapsedMs = now - (game.startedAt || now);
if (currentHeight > user.blockRec && elapsedMs >= graceMs) {
// Keep this check immediately adjacent to terminal mutation. The synchronous
// beginShutdown flag is the authoritative cut between snapshot/restart work.
if (this._isShuttingDown) break;
if (this.debugManager.CONSOLE_LOGGING) {
console.log(`💀 GAME TIMEOUT for player ${socketId}: entered on block ${user.blockRec}, died on block ${currentHeight}`);
}
game.gameState = 'lost';
const fakeSocket = { id: socketId };
try {
await this.handleGameOver(fakeSocket, game, 'lost', 'timeout', 'You didn\'t escape before the block time limit!');
} catch (e) {
console.error(`checkGamesTimeout: handleGameOver failed for ${socketId}:`, e.message);
}
}
}
}
});
}
// Payment system handlers (placeholder for compatibility)
@@ -1686,6 +1949,11 @@ class SocketHandlers {
console.log('🛑 SocketHandlers shutting down...');
}
// Let work that crossed the synchronous admission boundary reach a stable state before
// freezing the settlement set. This includes a block-timeout pass paused on an earlier
// game's durable completion.
await this.drainShutdownProducers();
// Settle terminal solo runs while the DB pool is still open. This is idempotent if the
// top-level graceful-shutdown path already drained them.
if (this.gameManager?.shutdown) {
+59 -29
View File
@@ -10,8 +10,6 @@
* to seamlessly resume their session when they reconnect.
*/
const { normalizeError } = require('../utils/errors');
class SuspendedGameManager {
constructor({ debugManager, activeGames, cleanupTimeoutMs = 300000 }) {
this.debugManager = debugManager;
@@ -70,8 +68,14 @@ class SuspendedGameManager {
this.suspendedGames.set(dbUserId, suspendedState);
this.stats.gamesSuspended++;
// Set cleanup timer
this._scheduleCleanup(dbUserId);
// Ordinary network disconnects retain the existing bounded in-memory grace period.
// A durable restart snapshot is different: PostgreSQL is the authority and the process
// must keep the reconstructed object available until its owner reconnects (or an operator
// explicitly resolves the active row). Expiring only the Map entry would strand the
// durable active game and silently make it unresumable.
if (additionalState.durableRestartSnapshot !== true) {
this._scheduleCleanup(dbUserId);
}
if (this.debugManager?.CONSOLE_LOGGING) {
console.log(`🔄 [SuspendedGameManager] Suspended game ${game.id} for user ${dbUserId}`);
@@ -106,17 +110,14 @@ class SuspendedGameManager {
* @param {number} dbUserId - Database user ID
* @param {string} newSocketId - The new socket ID
* @param {Object} newUser - The new User object
* @returns {Object|null} The restored game or null if no suspended game
* @returns {Promise<Object|null>} The restored game or null if no suspended game
*/
restoreGame(dbUserId, newSocketId, newUser) {
async restoreGame(dbUserId, newSocketId, newUser) {
const suspended = this.suspendedGames.get(dbUserId);
if (!suspended) {
return null;
}
// Cancel cleanup timer
this._cancelCleanup(dbUserId);
const game = suspended.game;
// A settlement retry may have committed after disconnect but before reconnect. Never
@@ -126,6 +127,55 @@ class SuspendedGameManager {
return null;
}
// Claim a durable restart snapshot and realign the socket in one PostgreSQL statement
// before exposing the game as playable. If the row is no longer exactly active for this
// game/user/seed, no snapshot is consumed and no in-memory state is revived.
const db = game.db || suspended.db || null;
if (suspended.durableRestartSnapshot === true) {
if (!db || !game.dbId || !game.id) {
throw new Error('Durable solo restart snapshot is missing its database identity');
}
const claimed = await db.query(`
WITH valid_game AS MATERIALIZED (
SELECT id
FROM games
WHERE id = $1 AND user_id = $2 AND dungeon_seed = $3
AND status = 'active' AND completed_at IS NULL
FOR UPDATE
), consumed AS (
DELETE FROM solo_restart_snapshots s
USING valid_game g
WHERE s.game_id = g.id AND s.user_id = $2 AND s.snapshot_version = $4
RETURNING s.game_id
)
UPDATE games
SET socket_id = $5
WHERE id = $1 AND EXISTS (SELECT 1 FROM consumed)
RETURNING id
`, [game.dbId, dbUserId, game.id, suspended.snapshotVersion, newSocketId]);
if (claimed.rowCount !== 1) {
const error = new Error('Durable solo restart snapshot could not be claimed');
error.code = 'SOLO_RESTART_SNAPSHOT_CLAIM_FAILED';
throw error;
}
} else if (db && game.id) {
const aligned = await db.query(
`UPDATE games SET socket_id = $1
WHERE dungeon_seed = $2 AND status = 'active' AND completed_at IS NULL
RETURNING id`,
[newSocketId, game.id]
);
if (aligned.rowCount !== 1) {
const error = new Error('Suspended solo game is no longer active');
error.code = 'SUSPENDED_GAME_REALIGN_FAILED';
throw error;
}
}
// Only mutate the cache/object after the durable claim succeeds. A transient database
// failure therefore leaves the suspended entry intact for a later reconnect attempt.
this._cancelCleanup(dbUserId);
// Update game with new socket ID
game.socketId = newSocketId;
@@ -135,26 +185,6 @@ class SuspendedGameManager {
if (game.userId == null && game.dbUserId != null) game.userId = game.dbUserId;
if (game.dbUserId == null && game.userId != null) game.dbUserId = game.userId;
// Align the persisted game row's socket_id to the reconnecting socket so
// downstream row-matching (game-over UPDATE, completion) stays consistent.
// Fire-and-forget so restoreGame remains synchronous for its callers.
try {
const db = game.db || suspended.db || null;
if (db && game.id) {
Promise.resolve(
db.query('UPDATE games SET socket_id = $1 WHERE dungeon_seed = $2', [newSocketId, game.id])
).catch((err) => {
if (this.debugManager?.CONSOLE_LOGGING) {
console.warn(`[SuspendedGameManager] socket_id realign failed for game ${game.id}: ${err.message}`);
}
});
}
} catch (err) {
if (this.debugManager?.CONSOLE_LOGGING) {
console.warn(`[SuspendedGameManager] socket_id realign threw for game ${game.id}: ${normalizeError(err).message}`);
}
}
// Update the user reference if provided
if (newUser) {
game.user = newUser;
+1 -1
View File
@@ -15,7 +15,7 @@ const { Pool } = require('pg');
const DatabaseManager = require('../db/databaseManager');
const CONFIRM_PREFIX = 'MIGRATE_DISPOSABLE_CLONE:';
const MINIMUM_RELEASE_MIGRATION = 42;
const MINIMUM_RELEASE_MIGRATION = 43;
const MIGRATIONS_PATH = path.join(__dirname, '..', 'migrations');
const DATABASE_NAME = /^[a-z][a-z0-9_]{2,62}$/;
const DATABASE_USER = /^[a-z_][a-z0-9_]{0,62}$/;
+2 -1
View File
@@ -34,7 +34,8 @@ const REQUIRED_MIGRATIONS = Object.freeze([
'039_unresolved_solo_payout_review.sql',
'040_paid_match_entropy_precommit.sql',
'041_financial_event_outbox.sql',
'042_immutable_financial_event_snapshots.sql'
'042_immutable_financial_event_snapshots.sql',
'043_durable_solo_restart_snapshots.sql'
]);
const SCENARIOS = Object.freeze({