From bd63785cb02b1a37886d9c0b8b38a9195d9c13f6 Mon Sep 17 00:00:00 2001 From: Alexander Blair Date: Wed, 17 May 2017 07:07:31 -0700 Subject: [PATCH 01/17] Update to start working out the env_sync segfaults. --- lib/local_comms.js | 6 ++++-- lib/worker.js | 2 ++ package.json | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/local_comms.js b/lib/local_comms.js index 493bde7..9e52b4f 100644 --- a/lib/local_comms.js +++ b/lib/local_comms.js @@ -18,9 +18,9 @@ function Database(){ path: global.config.db_storage_path, maxDbs: 10, mapSize: 24 * 1024 * 1024 * 1024, - noSync: true, + noSync: false, mapAsync: true, - useWritemap: true, + useWritemap: false, noMetaSync: true, maxReaders: 512 }); @@ -477,12 +477,14 @@ function Database(){ this.refreshEnv = function(){ if (this.dirtyenv === true){ + /* console.log("Database Worker: Reloading LMDB Env"); global.database.env.sync(function(){ }); clearInterval(this.intervalID); global.database.env.close(); this.initEnv(); + */ } }; diff --git a/lib/worker.js b/lib/worker.js index f8c0486..6b0a171 100644 --- a/lib/worker.js +++ b/lib/worker.js @@ -449,6 +449,8 @@ function updateBlockHeader() { value: body.result.block_header.reward, ts: body.result.block_header.timestamp })]); + } else if (body.result.block_header.hash === lastBlockHash) { + console.debug("Block headers identical to historical header. Ignoring"); } else { console.error("GetLastBlockHeader Error during block header update"); } diff --git a/package.json b/package.json index 0c3842f..db30ebe 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "moment": "2.17.1", "multi-hashing": "git+https://github.com/Snipa22/node-multi-hashing-aesni.git", "mysql": "2.12.0", - "node-lmdb": "^0.4.3", + "node-lmdb": "0.4.12", "promise-mysql": "3.0.0", "protocol-buffers": "^3.2.1", "range": "0.0.3", From db1a3263e4b9de25c42df984ef89e7a0d00d25ac Mon Sep 17 00:00:00 2001 From: Alexander Blair Date: Wed, 17 May 2017 07:10:26 -0700 Subject: [PATCH 02/17] Fixing worker so it doesn't break so bad. --- lib/worker.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/worker.js b/lib/worker.js index 6b0a171..e91ab39 100644 --- a/lib/worker.js +++ b/lib/worker.js @@ -450,7 +450,7 @@ function updateBlockHeader() { ts: body.result.block_header.timestamp })]); } else if (body.result.block_header.hash === lastBlockHash) { - console.debug("Block headers identical to historical header. Ignoring"); + console.log("Block headers identical to historical header. Ignoring"); } else { console.error("GetLastBlockHeader Error during block header update"); } From cce2a67c70452ccaa1825928004168e543d9a46c Mon Sep 17 00:00:00 2001 From: Alexander Blair Date: Fri, 19 May 2017 14:51:48 -0700 Subject: [PATCH 03/17] New cache function for bulk-inserts in single txn to avoid excessive disk writes and blocking by worker. Adding new getBlockByID function for future tool functionality. --- lib/local_comms.js | 25 +++++++++++++++++++++++++ lib/worker.js | 12 +++++++----- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/lib/local_comms.js b/lib/local_comms.js index 9e52b4f..714e228 100644 --- a/lib/local_comms.js +++ b/lib/local_comms.js @@ -218,6 +218,21 @@ function Database(){ return highestBlock; }; + this.getBlockByID = function(blockID){ + this.refreshEnv(); + debug("Getting the data for blockID: " + blockID); + let txn = this.env.beginTxn({readOnly: true}); + let data = txn.getBinary(blockID); + if (data === null){ + debug("Unable to get block at height: "+ blockID); + return false; + } + let blockData = global.protos.Block.decode(data); + txn.commit(); + debug("Done getting the last block for: "+ blockData.poolType + " height of: "+ blockID); + return blockData; + }; + this.calculateShares = function(blockData, blockHeight){ debug("Calculating shares for "+ blockData.hash); this.refreshEnv(); @@ -376,6 +391,16 @@ function Database(){ txn.commit(); }; + this.bulkSetCache = function(cacheUpdates){ + let txn = this.env.beginTxn(); + for (let key in cacheUpdates){ + if (cacheUpdates.hasOwnProperty(key)){ + txn.putString(this.cacheDB, key, JSON.stringify(cacheUpdates[key])); + } + } + txn.commit(); + }; + this.cleanShareDB = function() { /* This function takes the difficulty of the current block, and the last PPS block. If it's 0, save everything, diff --git a/lib/worker.js b/lib/worker.js index e91ab39..8e31caf 100644 --- a/lib/worker.js +++ b/lib/worker.js @@ -137,6 +137,7 @@ function updateShareStats() { if (globalMinerList === false) { globalMinerList = []; } + let cache_updates = {}; // pplns: 0, pps: 0, solo: 0, prop: 0, global: 0 ['pplns', 'pps', 'solo', 'prop', 'global'].forEach(function (key) { let cachedData = global.database.getCache(key + "_stats"); @@ -172,7 +173,7 @@ function updateShareStats() { minerHistory: [{ts: currentTime, cn: cachedData.hash}] }; } - global.database.setCache(key + "_stats", cachedData); + cache_updates[key + "_stats"] = cachedData; }); minerList.forEach(function (miner) { if (globalMinerList.indexOf(miner) === -1) { @@ -206,7 +207,7 @@ function updateShareStats() { badShares: 0 }; } - global.database.setCache(miner, cachedData); + cache_updates[miner] = cachedData; }); globalMinerList.forEach(function (miner) { if (minerList.indexOf(miner) === -1) { @@ -234,14 +235,15 @@ function updateShareStats() { }); } minerStats.hash = 0; - global.database.setCache(miner, minerStats); + cache_updates[miner] = minerStats; } } }); Object.keys(identifiers).forEach(function (key) { - global.database.setCache(key + '_identifiers', identifiers[key]); + cache_updates[key + '_identifiers'] = identifiers[key]; }); - global.database.setCache('minerList', globalMinerList); + cache_updates['minerList'] = globalMinerList; + global.database.bulkSetCache(cache_updates); callback(null); }); } From cdf17d4e7097303ce6e3f0850eb3b8676213c140 Mon Sep 17 00:00:00 2001 From: Alexander Blair Date: Fri, 19 May 2017 16:40:22 -0700 Subject: [PATCH 04/17] Update to try and bulk-insert on new shares. We'll see how it goes.... --- lib/local_comms.js | 108 +++++++++++++++++++++++++++++++++++++++++++++ lib/remoteShare.js | 26 +++++++---- 2 files changed, 126 insertions(+), 8 deletions(-) diff --git a/lib/local_comms.js b/lib/local_comms.js index 714e228..3ca0829 100644 --- a/lib/local_comms.js +++ b/lib/local_comms.js @@ -179,6 +179,114 @@ function Database(){ callback(true); }; + this.storeBulkShares = function(shareObject) { + let cachedData = { + global_stats: global.database.getCache('global_stats'), + pplns_stats: global.database.getCache('pplns_stats'), + pps_stats: global.database.getCache('pps_stats'), + solo_stats: global.database.getCache('solo_stats') + }; + let shares = {}; // Shares keyed by blockID + let shareCount = 0; + shareObject.forEach(function(data){ + //Data is the share object at this point. + let share = global.protos.Share.decode(data); + if (!shares.hasOwnProperty(share.blockHeight)){ + shares[share.blockHeight] = []; + } + shares[share.blockHeight].push(data); + shareCount += 1; + let minerID = share.paymentAddress; + if (typeof(share.paymentID) !== 'undefined' && share.paymentID.length > 10) { + minerID = minerID + '.' + share.paymentID; + } + let minerIDWithIdentifier = minerID + "_" + share.identifier; + if (!cachedData.global_stats.hasOwnProperty('totalHashes')){ + cachedData.global_stats.totalHashes = 0; + } + if (!cachedData.global_stats.hasOwnProperty('roundHashes')){ + cachedData.global_stats.roundHashes = 0; + } + cachedData.global_stats.totalHashes += share.shares; + cachedData.global_stats.roundHashes += share.shares; + let stats_type = 'pplns_stats'; + switch (share.poolType) { + case global.protos.POOLTYPE.PPLNS: + stats_type = 'pplns_stats'; + break; + case global.protos.POOLTYPE.PPS: + stats_type = 'pps_stats'; + break; + case global.protos.POOLTYPE.SOLO: + stats_type = 'solo_stats'; + break; + } + if (!cachedData[stats_type].hasOwnProperty('totalHashes')){ + cachedData[stats_type].totalHashes = 0; + } + if (!cachedData[stats_type].hasOwnProperty('roundHashes')){ + cachedData[stats_type].roundHashes = 0; + } + cachedData[stats_type].totalHashes += share.shares; + cachedData[stats_type].roundHashes += share.shares; + if (!cachedData.hasOwnProperty(minerID)){ + let minerCache = global.database.getCache(minerID); + if (minerCache === false){ + minerCache = {totalHashes: 0, goodShares: 0}; + } + cachedData[minerID] = minerCache; + } + if (!cachedData.hasOwnProperty(minerIDWithIdentifier)){ + let minerCache = global.database.getCache(minerIDWithIdentifier); + if (minerCache === false){ + minerCache = {totalHashes: 0, goodShares: 0}; + } + cachedData[minerIDWithIdentifier] = minerCache; + } + if (!cachedData[minerIDWithIdentifier].hasOwnProperty('totalHashes')){ + cachedData[minerIDWithIdentifier].totalHashes = 0; + } + if (!cachedData[minerIDWithIdentifier].hasOwnProperty('goodShares')){ + cachedData[minerIDWithIdentifier].goodShares = 0; + } + if (!cachedData[minerID].hasOwnProperty('totalHashes')){ + cachedData[minerID].totalHashes = 0; + } + if (!cachedData[minerID].hasOwnProperty('goodShares')){ + cachedData[minerID].goodShares = 0; + } + cachedData[minerIDWithIdentifier].totalHashes += share.shares; + cachedData[minerID].totalHashes += share.shares; + cachedData[minerIDWithIdentifier].goodShares += 1; + cachedData[minerID].goodShares += 1; + if(shareObject.length === shareCount){ + // Perform insert and return + let txn = global.database.env.beginTxn(); + for (let key in cachedData){ + if (cachedData.hasOwnProperty(key)){ + txn.putString(global.database.cacheDB, key, JSON.stringify(cachedData[key])); + } + } + let blocksSeen = 0; + for (let key in shares){ + if (shares.hasOwnProperty(key)){ + blocksSeen += 1; + let sharesSeen = 0; + shares[key].forEach(function(share){ + sharesSeen += 1; + txn.putBinary(global.database.shareDB, key, share); + if (Object.keys(shares).length === blocksSeen && sharesSeen === shares[key].length){ + txn.commit(); + return true; + } + }); + } + } + } + }); + return false; + }; + this.storeInvalidShare = function(shareData, callback){ try { let share = global.protos.InvalidShare.decode(shareData); diff --git a/lib/remoteShare.js b/lib/remoteShare.js index dd291f1..ed02ae8 100644 --- a/lib/remoteShare.js +++ b/lib/remoteShare.js @@ -12,6 +12,7 @@ if (cluster.isMaster) { } else { threadName = "(Worker " + cluster.worker.id + " - " + process.pid + ") "; } +let shareData = []; // Websocket Stuffs. app.use(function(req, res, next){ @@ -21,6 +22,13 @@ app.use(function(req, res, next){ })); }); +// Master/Slave communication Handling +function messageHandler(message) { + shareData.push(message.data); +} + +process.on('message', messageHandler); + app.post('/leafApi', function (req, res) { try { let msgData = global.protos.WSData.decode(req.body); @@ -29,14 +37,8 @@ app.post('/leafApi', function (req, res) { } switch (msgData.msgType) { case global.protos.MESSAGETYPE.SHARE: - global.database.storeShare(msgData.exInt, msgData.msg, function(data){ - if (!data){ - return res.status(400).end(); - } else { - return res.json({'success': true}); - } - }); - break; + process.send({block: msgData.exInt, data: msgData.msg}); + return res.json({'success': true}); case global.protos.MESSAGETYPE.BLOCK: global.database.storeBlock(msgData.exInt, msgData.msg, function(data){ if (!data){ @@ -68,6 +70,14 @@ if (cluster.isMaster) { let numWorkers = require('os').cpus().length; console.log('Master cluster setting up ' + numWorkers + ' workers...'); + setInterval(function(){ + if (Object.keys(shareData).length > 0){ + if (global.database.storeBulkShares(shareData)){ + shareData = []; + } + } + }, 1000); + for (let i = 0; i < numWorkers; i++) { let worker = cluster.fork(); workerList.push(worker); From 063b206d46b32ba9a58a3cc73d54b548b49a8068 Mon Sep 17 00:00:00 2001 From: Alexander Blair Date: Fri, 19 May 2017 16:54:20 -0700 Subject: [PATCH 05/17] Moving share validation into remoteShare before it ever gets to our internal processing DB loop. --- lib/local_comms.js | 4 +++- lib/remoteShare.js | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/local_comms.js b/lib/local_comms.js index 3ca0829..19829e8 100644 --- a/lib/local_comms.js +++ b/lib/local_comms.js @@ -188,14 +188,15 @@ function Database(){ }; let shares = {}; // Shares keyed by blockID let shareCount = 0; + debug(shareObject.length + ' shares to process'); shareObject.forEach(function(data){ //Data is the share object at this point. let share = global.protos.Share.decode(data); + shareCount += 1; if (!shares.hasOwnProperty(share.blockHeight)){ shares[share.blockHeight] = []; } shares[share.blockHeight].push(data); - shareCount += 1; let minerID = share.paymentAddress; if (typeof(share.paymentID) !== 'undefined' && share.paymentID.length > 10) { minerID = minerID + '.' + share.paymentID; @@ -276,6 +277,7 @@ function Database(){ sharesSeen += 1; txn.putBinary(global.database.shareDB, key, share); if (Object.keys(shares).length === blocksSeen && sharesSeen === shares[key].length){ + debug('Made it to where I can do the insert'); txn.commit(); return true; } diff --git a/lib/remoteShare.js b/lib/remoteShare.js index ed02ae8..4d9a905 100644 --- a/lib/remoteShare.js +++ b/lib/remoteShare.js @@ -37,7 +37,11 @@ app.post('/leafApi', function (req, res) { } switch (msgData.msgType) { case global.protos.MESSAGETYPE.SHARE: - process.send({block: msgData.exInt, data: msgData.msg}); + try { + global.protos.Share.decode(msgData.msg); + process.send({block: msgData.exInt, data: msgData.msg}); + } catch (e){ + } return res.json({'success': true}); case global.protos.MESSAGETYPE.BLOCK: global.database.storeBlock(msgData.exInt, msgData.msg, function(data){ @@ -80,6 +84,7 @@ if (cluster.isMaster) { for (let i = 0; i < numWorkers; i++) { let worker = cluster.fork(); + worker.on('message', messageHandler); workerList.push(worker); } @@ -91,6 +96,7 @@ if (cluster.isMaster) { console.log('Worker ' + worker.process.pid + ' died with code: ' + code + ', and signal: ' + signal); console.log('Starting a new worker'); worker = cluster.fork(); + worker.on('message', messageHandler); workerList.push(worker); }); } else { From 21ba152429e57cb253606918f4eae66767fcf8df Mon Sep 17 00:00:00 2001 From: Alexander Blair Date: Fri, 19 May 2017 17:08:47 -0700 Subject: [PATCH 06/17] Update with the new share handling system. --- lib/local_comms.js | 9 ++++----- lib/remoteShare.js | 5 ++--- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/lib/local_comms.js b/lib/local_comms.js index 19829e8..af64412 100644 --- a/lib/local_comms.js +++ b/lib/local_comms.js @@ -189,14 +189,13 @@ function Database(){ let shares = {}; // Shares keyed by blockID let shareCount = 0; debug(shareObject.length + ' shares to process'); - shareObject.forEach(function(data){ + shareObject.forEach(function(share){ //Data is the share object at this point. - let share = global.protos.Share.decode(data); shareCount += 1; if (!shares.hasOwnProperty(share.blockHeight)){ shares[share.blockHeight] = []; } - shares[share.blockHeight].push(data); + shares[share.blockHeight].push(share); let minerID = share.paymentAddress; if (typeof(share.paymentID) !== 'undefined' && share.paymentID.length > 10) { minerID = minerID + '.' + share.paymentID; @@ -273,9 +272,9 @@ function Database(){ if (shares.hasOwnProperty(key)){ blocksSeen += 1; let sharesSeen = 0; - shares[key].forEach(function(share){ + shares[key].forEach(function(final_share){ sharesSeen += 1; - txn.putBinary(global.database.shareDB, key, share); + txn.putBinary(global.database.shareDB, key, global.protos.Share.encode(final_share)); if (Object.keys(shares).length === blocksSeen && sharesSeen === shares[key].length){ debug('Made it to where I can do the insert'); txn.commit(); diff --git a/lib/remoteShare.js b/lib/remoteShare.js index 4d9a905..0dc884f 100644 --- a/lib/remoteShare.js +++ b/lib/remoteShare.js @@ -24,7 +24,7 @@ app.use(function(req, res, next){ // Master/Slave communication Handling function messageHandler(message) { - shareData.push(message.data); + shareData.push(message); } process.on('message', messageHandler); @@ -38,8 +38,7 @@ app.post('/leafApi', function (req, res) { switch (msgData.msgType) { case global.protos.MESSAGETYPE.SHARE: try { - global.protos.Share.decode(msgData.msg); - process.send({block: msgData.exInt, data: msgData.msg}); + process.send(global.protos.Share.decode(msgData.msg)); } catch (e){ } return res.json({'success': true}); From 245bc6454e7856f98b6e95608b5418dcf8ac54f4 Mon Sep 17 00:00:00 2001 From: Alexander Blair Date: Fri, 19 May 2017 17:10:30 -0700 Subject: [PATCH 07/17] Update with parseInt --- lib/local_comms.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/local_comms.js b/lib/local_comms.js index af64412..19583a9 100644 --- a/lib/local_comms.js +++ b/lib/local_comms.js @@ -274,7 +274,7 @@ function Database(){ let sharesSeen = 0; shares[key].forEach(function(final_share){ sharesSeen += 1; - txn.putBinary(global.database.shareDB, key, global.protos.Share.encode(final_share)); + txn.putBinary(global.database.shareDB, parseInt(key), global.protos.Share.encode(final_share)); if (Object.keys(shares).length === blocksSeen && sharesSeen === shares[key].length){ debug('Made it to where I can do the insert'); txn.commit(); From 2bbcc70ff10190111ab783c920e3ca4e98ab23ec Mon Sep 17 00:00:00 2001 From: Alexander Blair Date: Fri, 19 May 2017 17:16:26 -0700 Subject: [PATCH 08/17] New share handler again. With catch for error. --- lib/local_comms.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/local_comms.js b/lib/local_comms.js index 19583a9..733524b 100644 --- a/lib/local_comms.js +++ b/lib/local_comms.js @@ -274,7 +274,11 @@ function Database(){ let sharesSeen = 0; shares[key].forEach(function(final_share){ sharesSeen += 1; - txn.putBinary(global.database.shareDB, parseInt(key), global.protos.Share.encode(final_share)); + try { + txn.putBinary(global.database.shareDB, parseInt(key), global.protos.Share.encode(final_share)); + } catch (e) { + debug(final_share); + } if (Object.keys(shares).length === blocksSeen && sharesSeen === shares[key].length){ debug('Made it to where I can do the insert'); txn.commit(); From e6cf90a230f7112315fbfe7d1c5e78044c82775f Mon Sep 17 00:00:00 2001 From: Alexander Blair Date: Fri, 19 May 2017 17:21:50 -0700 Subject: [PATCH 09/17] Update to make sure we clear the shares every submit. --- lib/remoteShare.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/remoteShare.js b/lib/remoteShare.js index 0dc884f..450fb3d 100644 --- a/lib/remoteShare.js +++ b/lib/remoteShare.js @@ -75,9 +75,8 @@ if (cluster.isMaster) { setInterval(function(){ if (Object.keys(shareData).length > 0){ - if (global.database.storeBulkShares(shareData)){ - shareData = []; - } + global.database.storeBulkShares(shareData); + shareData = []; } }, 1000); From 7e800538d8cab6e34b0a8bb6d457b73ee82dac80 Mon Sep 17 00:00:00 2001 From: Alexander Blair Date: Fri, 19 May 2017 17:38:27 -0700 Subject: [PATCH 10/17] More data fixes.... --- lib/local_comms.js | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/local_comms.js b/lib/local_comms.js index 733524b..b8410ab 100644 --- a/lib/local_comms.js +++ b/lib/local_comms.js @@ -201,10 +201,11 @@ function Database(){ minerID = minerID + '.' + share.paymentID; } let minerIDWithIdentifier = minerID + "_" + share.identifier; - if (!cachedData.global_stats.hasOwnProperty('totalHashes')){ + // this.incrementCacheData('global_stats', [{location: 'totalHashes', value: share.shares}, {location: 'roundHashes', value: share.shares}]); + if (!cachedData.global_stats.hasOwnProperty('totalHashes') || cachedData.global_stats.totalHashes === null){ cachedData.global_stats.totalHashes = 0; } - if (!cachedData.global_stats.hasOwnProperty('roundHashes')){ + if (!cachedData.global_stats.hasOwnProperty('roundHashes') || cachedData.global_stats.roundHashes === null){ cachedData.global_stats.roundHashes = 0; } cachedData.global_stats.totalHashes += share.shares; @@ -221,10 +222,10 @@ function Database(){ stats_type = 'solo_stats'; break; } - if (!cachedData[stats_type].hasOwnProperty('totalHashes')){ + if (!cachedData[stats_type].hasOwnProperty('totalHashes') || cachedData[stats_type].totalHashes === null){ cachedData[stats_type].totalHashes = 0; } - if (!cachedData[stats_type].hasOwnProperty('roundHashes')){ + if (!cachedData[stats_type].hasOwnProperty('roundHashes') || cachedData[stats_type].roundHashes === null){ cachedData[stats_type].roundHashes = 0; } cachedData[stats_type].totalHashes += share.shares; @@ -243,16 +244,16 @@ function Database(){ } cachedData[minerIDWithIdentifier] = minerCache; } - if (!cachedData[minerIDWithIdentifier].hasOwnProperty('totalHashes')){ + if (!cachedData[minerIDWithIdentifier].hasOwnProperty('totalHashes') || cachedData[minerIDWithIdentifier].totalHashes === null){ cachedData[minerIDWithIdentifier].totalHashes = 0; } - if (!cachedData[minerIDWithIdentifier].hasOwnProperty('goodShares')){ + if (!cachedData[minerIDWithIdentifier].hasOwnProperty('goodShares') || cachedData[minerIDWithIdentifier].goodShares === null){ cachedData[minerIDWithIdentifier].goodShares = 0; } - if (!cachedData[minerID].hasOwnProperty('totalHashes')){ + if (!cachedData[minerID].hasOwnProperty('totalHashes') || cachedData[minerID].totalHashes === null){ cachedData[minerID].totalHashes = 0; } - if (!cachedData[minerID].hasOwnProperty('goodShares')){ + if (!cachedData[minerID].hasOwnProperty('goodShares') || cachedData[minerID].goodShares === null){ cachedData[minerID].goodShares = 0; } cachedData[minerIDWithIdentifier].totalHashes += share.shares; From 6413b8934554f9104dee751050f5bfa5ba2d60bd Mon Sep 17 00:00:00 2001 From: Alexander Blair Date: Fri, 19 May 2017 17:43:58 -0700 Subject: [PATCH 11/17] If share.shares is not a number, don't try to add it to the processing list. --- lib/local_comms.js | 132 +++++++++++++++++++++++---------------------- 1 file changed, 67 insertions(+), 65 deletions(-) diff --git a/lib/local_comms.js b/lib/local_comms.js index b8410ab..1473357 100644 --- a/lib/local_comms.js +++ b/lib/local_comms.js @@ -192,74 +192,76 @@ function Database(){ shareObject.forEach(function(share){ //Data is the share object at this point. shareCount += 1; - if (!shares.hasOwnProperty(share.blockHeight)){ - shares[share.blockHeight] = []; - } - shares[share.blockHeight].push(share); - let minerID = share.paymentAddress; - if (typeof(share.paymentID) !== 'undefined' && share.paymentID.length > 10) { - minerID = minerID + '.' + share.paymentID; - } - let minerIDWithIdentifier = minerID + "_" + share.identifier; - // this.incrementCacheData('global_stats', [{location: 'totalHashes', value: share.shares}, {location: 'roundHashes', value: share.shares}]); - if (!cachedData.global_stats.hasOwnProperty('totalHashes') || cachedData.global_stats.totalHashes === null){ - cachedData.global_stats.totalHashes = 0; - } - if (!cachedData.global_stats.hasOwnProperty('roundHashes') || cachedData.global_stats.roundHashes === null){ - cachedData.global_stats.roundHashes = 0; - } - cachedData.global_stats.totalHashes += share.shares; - cachedData.global_stats.roundHashes += share.shares; - let stats_type = 'pplns_stats'; - switch (share.poolType) { - case global.protos.POOLTYPE.PPLNS: - stats_type = 'pplns_stats'; - break; - case global.protos.POOLTYPE.PPS: - stats_type = 'pps_stats'; - break; - case global.protos.POOLTYPE.SOLO: - stats_type = 'solo_stats'; - break; - } - if (!cachedData[stats_type].hasOwnProperty('totalHashes') || cachedData[stats_type].totalHashes === null){ - cachedData[stats_type].totalHashes = 0; - } - if (!cachedData[stats_type].hasOwnProperty('roundHashes') || cachedData[stats_type].roundHashes === null){ - cachedData[stats_type].roundHashes = 0; - } - cachedData[stats_type].totalHashes += share.shares; - cachedData[stats_type].roundHashes += share.shares; - if (!cachedData.hasOwnProperty(minerID)){ - let minerCache = global.database.getCache(minerID); - if (minerCache === false){ - minerCache = {totalHashes: 0, goodShares: 0}; + if (typeof(share.shares) === "number") { + if (!shares.hasOwnProperty(share.blockHeight)) { + shares[share.blockHeight] = []; } - cachedData[minerID] = minerCache; - } - if (!cachedData.hasOwnProperty(minerIDWithIdentifier)){ - let minerCache = global.database.getCache(minerIDWithIdentifier); - if (minerCache === false){ - minerCache = {totalHashes: 0, goodShares: 0}; + shares[share.blockHeight].push(share); + let minerID = share.paymentAddress; + if (typeof(share.paymentID) !== 'undefined' && share.paymentID.length > 10) { + minerID = minerID + '.' + share.paymentID; } - cachedData[minerIDWithIdentifier] = minerCache; + let minerIDWithIdentifier = minerID + "_" + share.identifier; + // this.incrementCacheData('global_stats', [{location: 'totalHashes', value: share.shares}, {location: 'roundHashes', value: share.shares}]); + if (!cachedData.global_stats.hasOwnProperty('totalHashes') || cachedData.global_stats.totalHashes === null) { + cachedData.global_stats.totalHashes = 0; + } + if (!cachedData.global_stats.hasOwnProperty('roundHashes') || cachedData.global_stats.roundHashes === null) { + cachedData.global_stats.roundHashes = 0; + } + cachedData.global_stats.totalHashes += share.shares; + cachedData.global_stats.roundHashes += share.shares; + let stats_type = 'pplns_stats'; + switch (share.poolType) { + case global.protos.POOLTYPE.PPLNS: + stats_type = 'pplns_stats'; + break; + case global.protos.POOLTYPE.PPS: + stats_type = 'pps_stats'; + break; + case global.protos.POOLTYPE.SOLO: + stats_type = 'solo_stats'; + break; + } + if (!cachedData[stats_type].hasOwnProperty('totalHashes') || cachedData[stats_type].totalHashes === null) { + cachedData[stats_type].totalHashes = 0; + } + if (!cachedData[stats_type].hasOwnProperty('roundHashes') || cachedData[stats_type].roundHashes === null) { + cachedData[stats_type].roundHashes = 0; + } + cachedData[stats_type].totalHashes += share.shares; + cachedData[stats_type].roundHashes += share.shares; + if (!cachedData.hasOwnProperty(minerID)) { + let minerCache = global.database.getCache(minerID); + if (minerCache === false) { + minerCache = {totalHashes: 0, goodShares: 0}; + } + cachedData[minerID] = minerCache; + } + if (!cachedData.hasOwnProperty(minerIDWithIdentifier)) { + let minerCache = global.database.getCache(minerIDWithIdentifier); + if (minerCache === false) { + minerCache = {totalHashes: 0, goodShares: 0}; + } + cachedData[minerIDWithIdentifier] = minerCache; + } + if (!cachedData[minerIDWithIdentifier].hasOwnProperty('totalHashes') || cachedData[minerIDWithIdentifier].totalHashes === null) { + cachedData[minerIDWithIdentifier].totalHashes = 0; + } + if (!cachedData[minerIDWithIdentifier].hasOwnProperty('goodShares') || cachedData[minerIDWithIdentifier].goodShares === null) { + cachedData[minerIDWithIdentifier].goodShares = 0; + } + if (!cachedData[minerID].hasOwnProperty('totalHashes') || cachedData[minerID].totalHashes === null) { + cachedData[minerID].totalHashes = 0; + } + if (!cachedData[minerID].hasOwnProperty('goodShares') || cachedData[minerID].goodShares === null) { + cachedData[minerID].goodShares = 0; + } + cachedData[minerIDWithIdentifier].totalHashes += share.shares; + cachedData[minerID].totalHashes += share.shares; + cachedData[minerIDWithIdentifier].goodShares += 1; + cachedData[minerID].goodShares += 1; } - if (!cachedData[minerIDWithIdentifier].hasOwnProperty('totalHashes') || cachedData[minerIDWithIdentifier].totalHashes === null){ - cachedData[minerIDWithIdentifier].totalHashes = 0; - } - if (!cachedData[minerIDWithIdentifier].hasOwnProperty('goodShares') || cachedData[minerIDWithIdentifier].goodShares === null){ - cachedData[minerIDWithIdentifier].goodShares = 0; - } - if (!cachedData[minerID].hasOwnProperty('totalHashes') || cachedData[minerID].totalHashes === null){ - cachedData[minerID].totalHashes = 0; - } - if (!cachedData[minerID].hasOwnProperty('goodShares') || cachedData[minerID].goodShares === null){ - cachedData[minerID].goodShares = 0; - } - cachedData[minerIDWithIdentifier].totalHashes += share.shares; - cachedData[minerID].totalHashes += share.shares; - cachedData[minerIDWithIdentifier].goodShares += 1; - cachedData[minerID].goodShares += 1; if(shareObject.length === shareCount){ // Perform insert and return let txn = global.database.env.beginTxn(); From e39ed5203cbb54d85c94be95f030c20fd2c732b3 Mon Sep 17 00:00:00 2001 From: Alexander Blair Date: Fri, 19 May 2017 17:47:36 -0700 Subject: [PATCH 12/17] Adding logging for errors in the share parsing. --- lib/local_comms.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/local_comms.js b/lib/local_comms.js index 1473357..61b8338 100644 --- a/lib/local_comms.js +++ b/lib/local_comms.js @@ -261,6 +261,8 @@ function Database(){ cachedData[minerID].totalHashes += share.shares; cachedData[minerIDWithIdentifier].goodShares += 1; cachedData[minerID].goodShares += 1; + } else { + console.error("Error in share parser: " + JSON.stringify(share)); } if(shareObject.length === shareCount){ // Perform insert and return From 6833ad7b0f6614d3afa9d3d915cc8a1879560d40 Mon Sep 17 00:00:00 2001 From: Alexander Blair Date: Fri, 19 May 2017 17:49:20 -0700 Subject: [PATCH 13/17] Adding a higher level filter in remoteShare --- lib/remoteShare.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/remoteShare.js b/lib/remoteShare.js index 450fb3d..60b5fff 100644 --- a/lib/remoteShare.js +++ b/lib/remoteShare.js @@ -24,7 +24,9 @@ app.use(function(req, res, next){ // Master/Slave communication Handling function messageHandler(message) { - shareData.push(message); + if (typeof message.shares === "number"){ + shareData.push(message); + } } process.on('message', messageHandler); From 753f2c8e814c2c77b73058790f3504e0aab4a011 Mon Sep 17 00:00:00 2001 From: Alexander Blair Date: Fri, 19 May 2017 18:08:06 -0700 Subject: [PATCH 14/17] Update to include the new tool framework. --- init.js | 94 +++++++++++++++++++++++++++++-------------------- tools/blocks.js | 43 ++++++++++++++++++++++ 2 files changed, 98 insertions(+), 39 deletions(-) create mode 100644 tools/blocks.js diff --git a/init.js b/init.js index 053cee8..2ba9f62 100644 --- a/init.js +++ b/init.js @@ -5,11 +5,13 @@ let argv = require('minimist')(process.argv.slice(2)); let config = fs.readFileSync("./config.json"); let coinConfig = fs.readFileSync("./coinConfig.json"); let protobuf = require('protocol-buffers'); +let path = require('path'); global.support = require("./lib/support.js")(); global.config = JSON.parse(config); global.mysql = mysql.createPool(global.config.mysql); global.protos = protobuf(fs.readFileSync('./lib/data.proto')); +global.argv = argv; let comms; let coinInc; @@ -52,46 +54,60 @@ global.mysql.query("SELECT * FROM config").then(function (rows) { global.database.initEnv(); global.coinFuncs.blockedAddresses.push(global.config.pool.address); global.coinFuncs.blockedAddresses.push(global.config.payout.feeAddress); - switch(argv.module){ - case 'pool': - global.config.ports = []; - global.mysql.query("SELECT * FROM port_config").then(function(rows){ - rows.forEach(function(row){ - row.hidden = row.hidden === 1; - row.ssl = row.ssl === 1; - global.config.ports.push({ - port: row.poolPort, - difficulty: row.difficulty, - desc: row.portDesc, - portType: row.portType, - hidden: row.hidden, - ssl: row.ssl + if (argv.hasOwnProperty('tool') && fs.existsSync('./tools/'+argv.tool+'.js')) { + require('./lib/'+argv.tool+'.js'); + } else if (argv.hasOwnProperty('module')){ + switch(argv.module){ + case 'pool': + global.config.ports = []; + global.mysql.query("SELECT * FROM port_config").then(function(rows){ + rows.forEach(function(row){ + row.hidden = row.hidden === 1; + row.ssl = row.ssl === 1; + global.config.ports.push({ + port: row.poolPort, + difficulty: row.difficulty, + desc: row.portDesc, + portType: row.portType, + hidden: row.hidden, + ssl: row.ssl + }); }); + }).then(function(){ + require('./lib/pool.js'); }); - }).then(function(){ - require('./lib/pool.js'); - }); - break; - case 'blockManager': - require('./lib/blockManager.js'); - break; - case 'payments': - require('./lib/payments.js'); - break; - case 'api': - require('./lib/api.js'); - break; - case 'remoteShare': - require('./lib/remoteShare.js'); - break; - case 'worker': - require('./lib/worker.js'); - break; - case 'longRunner': - require('./lib/longRunner.js'); - break; - default: - console.error("Invalid module provided. Please provide a valid module"); - process.exit(1); + break; + case 'blockManager': + require('./lib/blockManager.js'); + break; + case 'payments': + require('./lib/payments.js'); + break; + case 'api': + require('./lib/api.js'); + break; + case 'remoteShare': + require('./lib/remoteShare.js'); + break; + case 'worker': + require('./lib/worker.js'); + break; + case 'longRunner': + require('./lib/longRunner.js'); + break; + default: + console.error("Invalid module provided. Please provide a valid module"); + process.exit(1); + } + } else { + console.error("Invalid module/tool provided. Please provide a valid module/tool"); + console.error("Valid Modules: pool, blockManager, payments, api, remoteShare, worker, longRunner"); + let valid_tools = "Valid Tools: "; + fs.readdirSync('./tools/').forEach(function(line){ + valid_tools += path.parse(line).name + ", "; + }); + valid_tools = valid_tools.slice(0, -2); + console.error(valid_tools); + process.exit(1); } }); \ No newline at end of file diff --git a/tools/blocks.js b/tools/blocks.js new file mode 100644 index 0000000..d6b2e1c --- /dev/null +++ b/tools/blocks.js @@ -0,0 +1,43 @@ +const valid_actions = ['finder', 'stats', 'list']; +let error = 0; +if (!global.argv.hasOwnProperty('action') || !valid_actions.hasOwnProperty(global.argv.action)) { + console.error("No action provided to block module."); + console.error("Valid actions: " + valid_actions.join(', ')); +} + +switch (global.argv.action) { + case 'finder': + if (!global.argv.hasOwnProperty('value')) { + console.error('No block provided in value field. Please use --value=blockID'); + error = 1; + break; + } + let blockID = parseInt(global.argv.value); + let block_data = global.database.getBlockByID(blockID); + /* + required string hash = 1; + required int64 difficulty = 2; + required int64 shares = 3; + required int64 timestamp = 4; + required POOLTYPE poolType = 5; + required bool unlocked = 6; + required bool valid = 7; + optional int64 value = 8; + */ + if (!block_data) { + console.error("Invalid blockID provided."); + error = 1; + break; + } + console.log("Data for block: " + blockID + '\n' + + 'Hash: ' + block_data.hash + '\n' + + 'Difficulty: ' + block_data.hash + '\n' + + 'Hashes Required: ' + block_data.hash + '\n' + + 'Find Time: ' + block_data.hash + '\n' + + 'Pool Type: ' + block_data.hash + '\n' + + 'Unlocked: ' + block_data.hash + '\n' + + 'Valid: ' + block_data.hash + '\n' + + 'Value: ' + block_data.hash + '\n'); +} + +process.exit(error); \ No newline at end of file From d058c0c7fd892516a6d5dcbcb1d8d1d9e6d74289 Mon Sep 17 00:00:00 2001 From: Alexander Blair Date: Fri, 19 May 2017 18:12:19 -0700 Subject: [PATCH 15/17] Fixing path to loading tools. --- init.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init.js b/init.js index 2ba9f62..1a2f257 100644 --- a/init.js +++ b/init.js @@ -55,7 +55,7 @@ global.mysql.query("SELECT * FROM config").then(function (rows) { global.coinFuncs.blockedAddresses.push(global.config.pool.address); global.coinFuncs.blockedAddresses.push(global.config.payout.feeAddress); if (argv.hasOwnProperty('tool') && fs.existsSync('./tools/'+argv.tool+'.js')) { - require('./lib/'+argv.tool+'.js'); + require('./tools/'+argv.tool+'.js'); } else if (argv.hasOwnProperty('module')){ switch(argv.module){ case 'pool': From d618eecd3b421ae1cb9e2903dfc8c025ad67b7f2 Mon Sep 17 00:00:00 2001 From: Alexander Blair Date: Fri, 19 May 2017 18:15:44 -0700 Subject: [PATCH 16/17] Need to refrence the right DB. --- lib/local_comms.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/local_comms.js b/lib/local_comms.js index 61b8338..da5d98a 100644 --- a/lib/local_comms.js +++ b/lib/local_comms.js @@ -340,7 +340,7 @@ function Database(){ this.refreshEnv(); debug("Getting the data for blockID: " + blockID); let txn = this.env.beginTxn({readOnly: true}); - let data = txn.getBinary(blockID); + let data = txn.getBinary(this.blockDB, blockID); if (data === null){ debug("Unable to get block at height: "+ blockID); return false; From ccc23c40c7435603574490575f76d774f640c542 Mon Sep 17 00:00:00 2001 From: Alexander Blair Date: Fri, 19 May 2017 18:17:26 -0700 Subject: [PATCH 17/17] Fixing a reference. --- tools/blocks.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/blocks.js b/tools/blocks.js index d6b2e1c..1de4149 100644 --- a/tools/blocks.js +++ b/tools/blocks.js @@ -1,6 +1,7 @@ const valid_actions = ['finder', 'stats', 'list']; let error = 0; -if (!global.argv.hasOwnProperty('action') || !valid_actions.hasOwnProperty(global.argv.action)) { + +if (!global.argv.hasOwnProperty('action') || valid_actions.indexOf(global.argv.action) === -1) { console.error("No action provided to block module."); console.error("Valid actions: " + valid_actions.join(', ')); }