From 9ee4b716516b685f0ea9e1d35e7f31c64c806a79 Mon Sep 17 00:00:00 2001 From: adaxi Date: Sun, 18 Feb 2018 22:08:01 +0100 Subject: [PATCH 1/3] Send an email when not able to get last block header --- lib/worker.js | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/lib/worker.js b/lib/worker.js index cb0b499..0aec1ce 100644 --- a/lib/worker.js +++ b/lib/worker.js @@ -521,6 +521,30 @@ function monitorNodes() { }); } +function monitorNodes() { + global.coinFuncs.getLastBlockHeader((err, block) => { + if (err !== null) { + return global.support.sendEmail( + global.config.general.adminEmail, + 'Failed to query daemon for last block header', + `The worker failed to return last block header. Please verify if the daemon is running properly.` + ); + } + const sql = 'SELECT blockID, hostname, ip FROM pools WHERE last_checkin > DATE_SUB(NOW(), INTERVAL 30 MINUTE)'; + global.mysql.query(sql).then(pools => { + pools.forEach(({ blockID, hostname, ip }) => { + if (blockID < block.height - 3) { + global.support.sendEmail( + global.config.general.adminEmail, + 'Pool server is behind in blocks', + `The pool server: ${hostname} with IP: ${ip} is ${(block.height - blockID)} blocks behind.` + ); + } + }) + }); + }); +} + updateShareStats(); updateBlockHeader(); updatePoolStats(); From 308f2f01353bc8d08f87a34214b297d49d38aeb9 Mon Sep 17 00:00:00 2001 From: adaxi Date: Sun, 18 Feb 2018 22:08:59 +0100 Subject: [PATCH 2/3] Remove old function --- lib/worker.js | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/lib/worker.js b/lib/worker.js index 0aec1ce..0723bc0 100644 --- a/lib/worker.js +++ b/lib/worker.js @@ -504,23 +504,6 @@ function updateWalletStats() { } -function monitorNodes() { - global.mysql.query("SELECT blockID, hostname, ip FROM pools WHERE last_checkin > date_sub(now(), interval 30 minute)").then(function (rows) { - global.coinFuncs.getLastBlockHeader(function (err, block) { - if (err !== null){ - console.error("Issue in getting block header. Skipping node monitor"); - return; - } - rows.forEach(function (row) { - if (row.blockID < block.height - 3) { - global.support.sendEmail(global.config.general.adminEmail, "Pool server behind in blocks", "The pool server: "+row.hostname+" with IP: "+row.ip+" is "+(block.height - row.blockID)+ " blocks behind"); - } - } - ); - }); - }); -} - function monitorNodes() { global.coinFuncs.getLastBlockHeader((err, block) => { if (err !== null) { From 774a5254cc4f57a26a8e450e17886935709c2190 Mon Sep 17 00:00:00 2001 From: adaxi Date: Sun, 18 Feb 2018 22:33:23 +0100 Subject: [PATCH 3/3] Avoid spamming the administrator when the daemon is stuck. --- lib/worker.js | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/lib/worker.js b/lib/worker.js index 0723bc0..1a0e545 100644 --- a/lib/worker.js +++ b/lib/worker.js @@ -504,13 +504,27 @@ function updateWalletStats() { } +let lastBlockCheckIsOk = true; function monitorNodes() { global.coinFuncs.getLastBlockHeader((err, block) => { if (err !== null) { - return global.support.sendEmail( + if (lastBlockCheckIsOk) { + lastBlockCheckIsOk = false; + global.support.sendEmail( + global.config.general.adminEmail, + 'Failed to query daemon for last block header', + `The worker failed to return last block header. Please verify if the daemon is running properly.` + ); + } + return + } + if (!lastBlockCheckIsOk) { + lastBlockCheckIsOk = true; + global.support.sendEmail( global.config.general.adminEmail, - 'Failed to query daemon for last block header', - `The worker failed to return last block header. Please verify if the daemon is running properly.` + 'Quering daemon for last block header is back to normal', + `An warning was sent to you indicating that the the worker failed to return the last block header. + The issue seems to be solved now.` ); } const sql = 'SELECT blockID, hostname, ip FROM pools WHERE last_checkin > DATE_SUB(NOW(), INTERVAL 30 MINUTE)'; @@ -525,6 +539,7 @@ function monitorNodes() { } }) }); + }); }