From 7fd28eec118cb9cff5f7de196c90d5e71f0b30f1 Mon Sep 17 00:00:00 2001 From: nzsquirrell Date: Sat, 26 Mar 2016 20:05:12 +1300 Subject: [PATCH] add support for address index --- src/init.cpp | 11 +++- src/main.cpp | 103 ++++++++++++++++++++++++++++++++++++-- src/main.h | 47 +++++++++++++++++ src/rpcclient.cpp | 1 + src/rpcrawtransaction.cpp | 59 ++++++++++++++++++++++ src/rpcserver.cpp | 1 + src/rpcserver.h | 1 + src/txdb.cpp | 48 ++++++++++++++++++ src/txdb.h | 4 ++ 9 files changed, 268 insertions(+), 7 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index f71fa4a1..dde36ac3 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -297,7 +297,8 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-sysperms", _("Create new files with system default permissions, instead of umask 077 (only effective with disabled wallet functionality)")); #endif strUsage += HelpMessageOpt("-txindex", strprintf(_("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)"), 0)); - + strUsage += HelpMessageOpt("-addrindex", strprintf(_("Maintain a full address index, used by the searchrawtransactions rpc call (default: %u)"), 0)); + strUsage += HelpMessageGroup(_("Connection options:")); strUsage += HelpMessageOpt("-addnode=", _("Add a node to connect to and attempt to keep the connection open")); strUsage += HelpMessageOpt("-banscore=", strprintf(_("Threshold for disconnecting misbehaving peers (default: %u)"), 100)); @@ -1124,7 +1125,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greated than nMaxDbcache int64_t nBlockTreeDBCache = nTotalCache / 8; - if (nBlockTreeDBCache > (1 << 21) && !GetBoolArg("-txindex", false)) + if (nBlockTreeDBCache > (1 << 21) && !GetBoolArg("-txindex", false) && !GetBoolArg("-addrindex", false)) nBlockTreeDBCache = (1 << 21); // block tree db cache shouldn't be larger than 2 MiB nTotalCache -= nBlockTreeDBCache; int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache @@ -1192,6 +1193,12 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) break; } + // Check for changed -addrindex state + if (fAddrIndex != GetBoolArg("-addrindex", false)) { + strLoadError = _("You need to rebuild the database using -reindex to change -addrindex"); + break; + } + uiInterface.InitMessage(_("Verifying blocks...")); if (fHavePruned && GetArg("-checkblocks", 288) > MIN_BLOCKS_TO_KEEP) { LogPrintf("Prune: pruned datadir may not have more than %d blocks; -checkblocks=%d may fail\n", diff --git a/src/main.cpp b/src/main.cpp index 2ea97397..a3109c0f 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -9,6 +9,7 @@ #include "alert.h" #include "arith_uint256.h" #include "auxpow.h" +#include "base58.h" #include "chainparams.h" #include "checkpoints.h" #include "checkqueue.h" @@ -57,6 +58,7 @@ bool fReindex = false; bool fTxIndex = false; bool fHavePruned = false; bool fPruneMode = false; +bool fAddrIndex = false; bool fIsBareMultisigStd = true; bool fCheckBlockIndex = false; bool fCheckpointsEnabled = true; @@ -1113,6 +1115,43 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa return true; } +bool ReadTransaction(CTransaction& tx, const CDiskTxPos &pos, uint256 &hashBlock) { + CAutoFile file(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION); + CBlockHeader header; + try { + file >> header; + fseek(file.Get(), pos.nTxOffset, SEEK_CUR); + file >> tx; + } catch (std::exception &e) { + return error("%s() : deserialize or I/O error", __PRETTY_FUNCTION__); + } + hashBlock = header.GetHash(); + return true; +} + +bool FindTransactionsByDestination(const CTxDestination &dest, std::set &setpos) { + uint160 addrid; + const CKeyID *pkeyid = boost::get(&dest); + if (pkeyid) + addrid = static_cast(*pkeyid); + if (addrid.IsNull()) { + const CScriptID *pscriptid = boost::get(&dest); + if (pscriptid) + addrid = static_cast(*pscriptid); + } + if (addrid.IsNull()) + return false; + + LOCK(cs_main); + if (!fAddrIndex) + return false; + std::vector vPos; + if (!pblocktree->ReadAddrIndex(addrid, vPos)) + return false; + setpos.insert(vPos.begin(), vPos.end()); + return true; +} + /** Return transaction in tx, and if it was found inside a block, its hash is placed in hashBlock */ bool GetTransaction(const uint256 &hash, CTransaction &txOut, uint256 &hashBlock, bool fAllowSlow) { @@ -1850,6 +1889,33 @@ static int64_t nTimeIndex = 0; static int64_t nTimeCallbacks = 0; static int64_t nTimeTotal = 0; +// Index either: a) every data push >=8 bytes, b) if no such pushes, the entire script +void static BuildAddrIndex(const CScript &script, const CExtDiskTxPos &pos, std::vector > &out) +{ + CScript::const_iterator pc = script.begin(); + CScript::const_iterator pend = script.end(); + std::vector data; + opcodetype opcode; + bool fHaveData = false; + while (pc < pend) { + script.GetOp(pc, opcode, data); + if (0 <= opcode && opcode <= OP_PUSHDATA4 && data.size() >= 8) { // data element + uint160 addrid; + if (data.size() <= 20) { + memcpy(&addrid, &data[0], data.size()); + } else { + addrid = Hash160(data); + } + out.push_back(std::make_pair(addrid, pos)); + fHaveData = true; + } + } + if (!fHaveData) { + uint160 addrid = Hash160(script); + out.push_back(std::make_pair(addrid, pos)); + } +} + bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, bool fJustCheck) { const CChainParams& chainparams = Params(); @@ -1925,9 +1991,13 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin CAmount nFees = 0; int nInputs = 0; unsigned int nSigOps = 0; - CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size())); - std::vector > vPos; - vPos.reserve(block.vtx.size()); + CExtDiskTxPos pos(CDiskTxPos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size())), pindex->nHeight); + std::vector > vPosTxid; + std::vector > vPosAddrid; + if (fTxIndex) + vPosTxid.reserve(block.vtx.size()); + if (fAddrIndex) + vPosAddrid.reserve(4*block.vtx.size()); blockundo.vtxundo.reserve(block.vtx.size() - 1); for (unsigned int i = 0; i < block.vtx.size(); i++) { @@ -1965,13 +2035,26 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin control.Add(vChecks); } + if (fTxIndex) + vPosTxid.push_back(std::make_pair(tx.GetHash(), pos)); + if (fAddrIndex) { + if (!tx.IsCoinBase()) { + BOOST_FOREACH(const CTxIn &txin, tx.vin) { + CCoins coins; + view.GetCoins(txin.prevout.hash, coins); + BuildAddrIndex(coins.vout[txin.prevout.n].scriptPubKey, pos, vPosAddrid); + } + } + BOOST_FOREACH(const CTxOut &txout, tx.vout) + BuildAddrIndex(txout.scriptPubKey, pos, vPosAddrid); + } + CTxUndo undoDummy; if (i > 0) { blockundo.vtxundo.push_back(CTxUndo()); } UpdateCoins(tx, state, view, i == 0 ? undoDummy : blockundo.vtxundo.back(), pindex->nHeight); - vPos.push_back(std::make_pair(tx.GetHash(), pos)); pos.nTxOffset += ::GetSerializeSize(tx, SER_DISK, CLIENT_VERSION); } int64_t nTime1 = GetTimeMicros(); nTimeConnect += nTime1 - nTimeStart; @@ -2012,8 +2095,12 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin } if (fTxIndex) - if (!pblocktree->WriteTxIndex(vPos)) + if (!pblocktree->WriteTxIndex(vPosTxid)) return AbortNode(state, "Failed to write transaction index"); + + if (fAddrIndex) + if (!pblocktree->AddAddrIndex(vPosAddrid)) + return AbortNode(state, "Failed to write address index"); // add this block to the view's block chain view.SetBestBlock(pindex->GetBlockHash()); @@ -3385,6 +3472,10 @@ bool static LoadBlockIndexDB() pblocktree->ReadFlag("txindex", fTxIndex); LogPrintf("%s: transaction index %s\n", __func__, fTxIndex ? "enabled" : "disabled"); + // Check whether we have an address index + pblocktree->ReadFlag("addrindex", fAddrIndex); + LogPrintf("%s: address index %s\n", __func__, fAddrIndex ? "enabled" : "disabled"); + // Load pointer to end of best chain BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); if (it == mapBlockIndex.end()) @@ -3543,6 +3634,8 @@ bool InitBlockIndex() { // Use the provided setting for -txindex in the new database fTxIndex = GetBoolArg("-txindex", false); pblocktree->WriteFlag("txindex", fTxIndex); + fAddrIndex = GetBoolArg("-addrindex", false); + pblocktree->WriteFlag("addrindex", fAddrIndex); LogPrintf("Initializing databases...\n"); // Only add the genesis block if not reindexing (in which case we reuse the one already on disk) diff --git a/src/main.h b/src/main.h index 97a13337..f5603992 100644 --- a/src/main.h +++ b/src/main.h @@ -110,6 +110,7 @@ extern bool fImporting; extern bool fReindex; extern int nScriptCheckThreads; extern bool fTxIndex; +extern bool fAddrIndex; extern bool fIsBareMultisigStd; extern bool fCheckBlockIndex; extern bool fCheckpointsEnabled; @@ -268,8 +269,52 @@ struct CDiskTxPos : public CDiskBlockPos CDiskBlockPos::SetNull(); nTxOffset = 0; } + + friend bool operator<(const CDiskTxPos &a, const CDiskTxPos &b) { + return (a.nFile < b.nFile || ( + (a.nFile == b.nFile) && (a.nPos < b.nPos || ( + (a.nPos == b.nPos) && (a.nTxOffset < b.nTxOffset))))); + } }; +struct CExtDiskTxPos : public CDiskTxPos +{ + unsigned int nHeight; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(*(CDiskTxPos*)this); + READWRITE(VARINT(nHeight)); + } + + CExtDiskTxPos(const CDiskTxPos &pos, int nHeightIn) : CDiskTxPos(pos), nHeight(nHeightIn) { + } + + CExtDiskTxPos() { + SetNull(); + } + + void SetNull() { + CDiskTxPos::SetNull(); + nHeight = 0; + } + + friend bool operator==(const CExtDiskTxPos &a, const CExtDiskTxPos &b) { + return (a.nHeight == b.nHeight && a.nFile == b.nFile && a.nPos == b.nPos && a.nTxOffset == b.nTxOffset); + } + + friend bool operator!=(const CExtDiskTxPos &a, const CExtDiskTxPos &b) { + return !(a == b); + } + + friend bool operator<(const CExtDiskTxPos &a, const CExtDiskTxPos &b) { + if (a.nHeight < b.nHeight) return true; + if (a.nHeight > b.nHeight) return false; + return ((const CDiskTxPos)a < (const CDiskTxPos)b); + } +}; CAmount GetMinRelayFee(const CTransaction& tx, unsigned int nBytes, bool fAllowFree); @@ -382,6 +427,8 @@ public: bool WriteBlockToDisk(CBlock& block, CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& messageStart); bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex); bool ReadBlockHeaderFromDisk(CBlockHeader& block, const CBlockIndex* pindex); +bool ReadTransaction(CTransaction& tx, const CDiskTxPos &pos, uint256 &hashBlock); +bool FindTransactionsByDestination(const CTxDestination &dest, std::set &setpos); /** Functions for validating blocks and updating the block tree */ diff --git a/src/rpcclient.cpp b/src/rpcclient.cpp index 4b576b37..19fd5c1b 100644 --- a/src/rpcclient.cpp +++ b/src/rpcclient.cpp @@ -75,6 +75,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "createrawtransaction", 1 }, { "signrawtransaction", 1 }, { "signrawtransaction", 2 }, + { "searchrawtransactions", 2 }, { "sendrawtransaction", 1 }, { "gettxout", 1 }, { "gettxout", 2 }, diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index 633f6df1..6204ad4e 100644 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -106,6 +106,65 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, Object& entry) } } +Value searchrawtransactions(const Array ¶ms, bool fHelp) +{ + if (fHelp || params.size() < 1 || params.size() > 4) + throw runtime_error("searchrawtransactions
[verbose=1] [skip=0] [count=100]\n"); + + if (!fAddrIndex) + throw JSONRPCError(RPC_MISC_ERROR, "Address index not enabled"); + + CBitcoinAddress address(params[0].get_str()); + if (!address.IsValid()) + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); + CTxDestination dest = address.Get(); + + std::set setpos; + if (!FindTransactionsByDestination(dest, setpos)) + throw JSONRPCError(RPC_DATABASE_ERROR, "Cannot search for address"); + + int nSkip = 0; + int nCount = 100; + bool fVerbose = true; + if (params.size() > 1) + fVerbose = (params[1].get_int() != 0); + if (params.size() > 2) + nSkip = params[2].get_int(); + if (params.size() > 3) + nCount = params[3].get_int(); + + if (nSkip < 0) + nSkip += setpos.size(); + if (nSkip < 0) + nSkip = 0; + if (nCount < 0) + nCount = 0; + + std::set::const_iterator it = setpos.begin(); + while (it != setpos.end() && nSkip--) it++; + + Array result; + while (it != setpos.end() && nCount--) { + CTransaction tx; + uint256 hashBlock; + if (!ReadTransaction(tx, *it, hashBlock)) + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Cannot read transaction from disk"); + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + ssTx << tx; + string strHex = HexStr(ssTx.begin(), ssTx.end()); + if (fVerbose) { + Object object; + TxToJSON(tx, hashBlock, object); + object.push_back(Pair("hex", strHex)); + result.push_back(object); + } else { + result.push_back(strHex); + } + it++; + } + return result; +} + Value getrawtransaction(const Array& params, bool fHelp) { if (fHelp || params.size() < 1 || params.size() > 2) diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 157f89f3..507b29f0 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -321,6 +321,7 @@ static const CRPCCommand vRPCCommands[] = { "rawtransactions", "decoderawtransaction", &decoderawtransaction, true }, { "rawtransactions", "decodescript", &decodescript, true }, { "rawtransactions", "getrawtransaction", &getrawtransaction, true }, + { "rawtransactions", "searchrawtransactions", &searchrawtransactions, true }, { "rawtransactions", "sendrawtransaction", &sendrawtransaction, false }, { "rawtransactions", "signrawtransaction", &signrawtransaction, false }, /* uses wallet if enabled */ diff --git a/src/rpcserver.h b/src/rpcserver.h index 276bb9e5..16638b75 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -211,6 +211,7 @@ extern json_spirit::Value setmocktime(const json_spirit::Array& params, bool fHe extern json_spirit::Value resendwallettransactions(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value getrawtransaction(const json_spirit::Array& params, bool fHelp); // in rcprawtransaction.cpp +extern json_spirit::Value searchrawtransactions(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value listunspent(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value lockunspent(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value listlockunspent(const json_spirit::Array& params, bool fHelp); diff --git a/src/txdb.cpp b/src/txdb.cpp index c0bfbdc1..ddc44f2f 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -78,6 +78,10 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { } CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CLevelDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) { + if (!Read('S', salt)) { + salt = GetRandHash(); + Write('S', salt); + } } bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) { @@ -176,6 +180,50 @@ bool CBlockTreeDB::WriteTxIndex(const std::vector return WriteBatch(batch); } +bool CBlockTreeDB::ReadAddrIndex(uint160 addrid, std::vector &list) { + boost::scoped_ptr pcursor(NewIterator()); + uint64_t lookupid; + { + CHashWriter ss(SER_GETHASH, 0); + ss << salt; + ss << addrid; + lookupid = UintToArith256(ss.GetHash()).GetLow64(); + } + CDataStream ssKeySet(SER_DISK, CLIENT_VERSION); + ssKeySet << make_pair('a', lookupid); + pcursor->Seek(ssKeySet.str()); + + while (pcursor->Valid()) { + std::pair, CExtDiskTxPos> key; + leveldb::Slice slKey = pcursor->key(); + try { + CDataStream ssKey(slKey.data(), slKey.data() + slKey.size(), SER_DISK, CLIENT_VERSION); + ssKey >> key; + } catch(std::exception &e) { + break; + } + if (key.first.first == 'a' && key.first.second == lookupid) { + list.push_back(key.second); + } else { + break; + } + pcursor->Next(); + } + return true; +} + +bool CBlockTreeDB::AddAddrIndex(const std::vector > &list) { + unsigned char foo[0]; + CLevelDBBatch batch; + for (std::vector >::const_iterator it=list.begin(); it!=list.end(); it++) { + CHashWriter ss(SER_GETHASH, 0); + ss << salt; + ss << it->first; + batch.Write(make_pair(make_pair('a', UintToArith256(ss.GetHash()).GetLow64()), it->second), FLATDATA(foo)); + } + return WriteBatch(batch, true); +} + bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) { return Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0'); } diff --git a/src/txdb.h b/src/txdb.h index bef5dc9f..767194c5 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -17,6 +17,7 @@ class CBlockFileInfo; class CBlockIndex; struct CDiskTxPos; +struct CExtDiskTxPos; class uint256; //! -dbcache default (MiB) @@ -47,6 +48,7 @@ class CBlockTreeDB : public CLevelDBWrapper public: CBlockTreeDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); private: + uint256 salt; CBlockTreeDB(const CBlockTreeDB&); void operator=(const CBlockTreeDB&); public: @@ -57,6 +59,8 @@ public: bool ReadReindexing(bool &fReindex); bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos); bool WriteTxIndex(const std::vector > &list); + bool ReadAddrIndex(uint160 addrid, std::vector &list); + bool AddAddrIndex(const std::vector > &list); bool WriteFlag(const std::string &name, bool fValue); bool ReadFlag(const std::string &name, bool &fValue); bool LoadBlockIndexGuts();