spacecruft
/
blockbook
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Change the way UTXO addresses are indexed - WIP

indexv2
Martin Boehm 2018-04-19 14:28:05 +02:00
parent 0ae9c446a0
commit c657381d7e
2 changed files with 151 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
db/rocksdb.go
View File

@ -263,7 +263,7 @@ func (d *RocksDB) writeAddressRecords(wb *gorocksdb.WriteBatch, block *bchain.Bl
func (d *RocksDB) addAddrIDToRecords(op int, wb *gorocksdb.WriteBatch, records map[string][]outpoint, addrID []byte, btxid []byte, vout int32, bh uint32) error {
if len(addrID) > 0 {
if len(addrID) > 1024 {
glog.Infof("block %d, skipping addrID of length %d", bh, len(addrID))
glog.Infof("rocksdb: block %d, skipping addrID of length %d", bh, len(addrID))
} else {
strAddrID := string(addrID)
records[strAddrID] = append(records[strAddrID], outpoint{
@ -305,18 +305,19 @@ func appendPackedAddrID(txAddrs []byte, addrID []byte, n uint32, remaining int)
}
func findAndRemoveUnspentAddr(unspentAddrs []byte, vout uint32) ([]byte, uint32, []byte) {
// the addresses are packed as lenaddrID:addrID:vout, where lenaddrID and vout are varints
for i := 0; i < len(unspentAddrs); {
l, lv1 := unpackVarint(unspentAddrs[i:])
// index of vout of address in unspentAddrs
j := i + int(l) + lv1
if j >= len(unspentAddrs) {
glog.Error("Inconsistent data in unspentAddrs")
glog.Error("rocksdb: Inconsistent data in unspentAddrs")
return nil, 0, unspentAddrs
}
n, lv2 := unpackVarint(unspentAddrs[j:])
if uint32(n) == vout {
addrID := append([]byte(nil), unspentAddrs[i+lv1:j]...)
unspentAddrs = append(unspentAddrs[:i], unspentAddrs[i+lv2:]...)
unspentAddrs = append(unspentAddrs[:i], unspentAddrs[j+lv2:]...)
return addrID, uint32(n), unspentAddrs
}
i += j + lv2
@ -325,7 +326,6 @@ func findAndRemoveUnspentAddr(unspentAddrs []byte, vout uint32) ([]byte, uint32,
}
func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error {
var err error
addresses := make(map[string][]outpoint)
unspentTxs := make(map[string][]byte)
btxIDs := make([][]byte, len(block.Txs))
@ -357,44 +357,35 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo
}
// locate unspent addresses and add them to addresses map them in format txid ^index
for txi, tx := range block.Txs {
btxID := btxIDs[txi]
// try to find the tx in current block
unspentAddrs, inThisBlock := unspentTxs[string(btxID)]
if !inThisBlock {
unspentAddrs, err = d.getUnspentTx(btxID)
spendingTxid := btxIDs[txi]
for i, input := range tx.Vin {
btxID, err := d.chainParser.PackTxid(input.Txid)
if err != nil {
return err
}
if unspentAddrs == nil {
glog.Warningf("rocksdb: height %d, tx %v in inputs but missing in unspentTxs", block.Height, tx.Txid)
continue
// try to find the tx in current block
unspentAddrs, inThisBlock := unspentTxs[string(btxID)]
if !inThisBlock {
unspentAddrs, err = d.getUnspentTx(btxID)
if err != nil {
return err
}
if unspentAddrs == nil {
glog.Warningf("rocksdb: height %d, tx %v in inputs but missing in unspentTxs", block.Height, tx.Txid)
continue
}
}
}
var addrID []byte
var n uint32
for _, input := range tx.Vin {
addrID, n, unspentAddrs = findAndRemoveUnspentAddr(unspentAddrs, input.Vout)
var addrID []byte
addrID, _, unspentAddrs = findAndRemoveUnspentAddr(unspentAddrs, input.Vout)
if addrID == nil {
glog.Warningf("rocksdb: height %d, tx %v vout %v in inputs but missing in unspentTxs", block.Height, tx.Txid, input.Vout)
continue
}
err = d.addAddrIDToRecords(op, wb, addresses, addrID, btxID, int32(^n), block.Height)
err = d.addAddrIDToRecords(op, wb, addresses, addrID, spendingTxid, int32(^i), block.Height)
if err != nil {
return err
}
}
if inThisBlock {
if len(unspentAddrs) == 0 {
delete(unspentTxs, string(btxID))
} else {
unspentTxs[string(btxID)] = unspentAddrs
}
} else {
if len(unspentAddrs) == 0 {
wb.DeleteCF(d.cfh[cfUnspentTxs], btxID)
} else {
wb.PutCF(d.cfh[cfUnspentTxs], btxID, unspentAddrs)
}
unspentTxs[string(btxID)] = unspentAddrs
}
}
if err := d.writeAddressRecords(wb, block, op, addresses); err != nil {
@ -404,7 +395,11 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo
for tx, val := range unspentTxs {
switch op {
case opInsert:
wb.PutCF(d.cfh[cfUnspentTxs], []byte(tx), val)
if len(val) == 0 {
wb.DeleteCF(d.cfh[cfUnspentTxs], []byte(tx))
} else {
wb.PutCF(d.cfh[cfUnspentTxs], []byte(tx), val)
}
case opDelete:
wb.DeleteCF(d.cfh[cfUnspentTxs], []byte(tx))
}
@ -722,6 +717,6 @@ func packVarint(i int32, buf []byte) int {
}
func unpackVarint(buf []byte) (int32, int) {
i, ofs := vlq.Uint(buf)
i, ofs := vlq.Int(buf)
return int32(i), ofs
}

124
db/rocksdb_test.go
View File

@ -6,6 +6,7 @@ import (
"encoding/hex"
"io/ioutil"
"os"
"sort"
"strconv"
"testing"
@ -50,6 +51,9 @@ type keyPair struct {
}
func checkColumn(d *RocksDB, col int, kp []keyPair) error {
sort.Slice(kp, func(i, j int) bool {
return kp[i].Key < kp[j].Key
})
it := d.db.NewIteratorCF(d.ro, d.cfh[col])
defer it.Close()
i := 0
@ -63,7 +67,7 @@ func checkColumn(d *RocksDB, col int, kp []keyPair) error {
}
val := hex.EncodeToString(it.Value().Data())
if val != kp[i].Value {
return errors.Errorf("Incorrect key %v found in column %v row %v, expecting %v", val, col, i, kp[i].Value)
return errors.Errorf("Incorrect value %v found in column %v row %v, expecting %v", val, col, i, kp[i].Value)
}
i++
}
@ -72,6 +76,15 @@ func checkColumn(d *RocksDB, col int, kp []keyPair) error {
}
return nil
}
// TestRocksDB_Index_UTXO is a composite test testing the whole indexing functionality for UTXO chains
// It does the following:
// 1) Connect two blocks (inputs from 2nd block are spending some outputs from the 1st block)
// 2) GetTransactions for known addresses
// 3) Disconnect block 2
// 4) GetTransactions for known addresses
// 5) Connect the block 2 back
// After each step, the whole content of DB is examined and any difference against expected state is regarded as failure
func TestRocksDB_Index_UTXO(t *testing.T) {
d := setupRocksDB(t, &btc.BitcoinParser{Params: btc.GetChainParams("test")})
defer closeAnddestroyRocksDB(t, d)
@ -155,4 +168,113 @@ func TestRocksDB_Index_UTXO(t *testing.T) {
}
}
// connect 2nd block - use some outputs from the 1st block as the inputs and 1 input uses tx from the same block
block2 := bchain.Block{
BlockHeader: bchain.BlockHeader{
Height: 225494,
Hash: "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6",
},
Txs: []bchain.Tx{
bchain.Tx{
Txid: "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25",
Vin: []bchain.Vin{
bchain.Vin{
Txid: "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75",
Vout: 0,
},
bchain.Vin{
Txid: "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840",
Vout: 1,
},
},
Vout: []bchain.Vout{
bchain.Vout{
N: 0,
ScriptPubKey: bchain.ScriptPubKey{
Hex: addressToPubKeyHex("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d),
},
},
bchain.Vout{
N: 1,
ScriptPubKey: bchain.ScriptPubKey{
Hex: addressToPubKeyHex("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d),
},
},
},
},
bchain.Tx{
Txid: "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71",
Vin: []bchain.Vin{
bchain.Vin{
Txid: "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25",
Vout: 0,
},
bchain.Vin{
Txid: "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75",
Vout: 1,
},
},
Vout: []bchain.Vout{
bchain.Vout{
N: 0,
ScriptPubKey: bchain.ScriptPubKey{
Hex: addressToPubKeyHex("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d),
},
},
bchain.Vout{
N: 1,
ScriptPubKey: bchain.ScriptPubKey{
Hex: addressToPubKeyHex("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d),
},
},
},
},
},
}
if err := d.ConnectBlock(&block2); err != nil {
t.Fatal(err)
}
if err := checkColumn(d, cfHeight, []keyPair{
keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997"},
keyPair{"000370d6", "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6"},
}); err != nil {
{
t.Fatal(err)
}
}
if err := checkColumn(d, cfAddresses, []keyPair{
keyPair{addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "00"},
keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02"},
keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00"},
keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02"},
keyPair{addressToPubKeyHex("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "00" + "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "01"},
keyPair{addressToPubKeyHex("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "02"},
keyPair{addressToPubKeyHex("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "00"},
keyPair{addressToPubKeyHex("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "02"},
keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "01"},
keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "03"},
keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "03"},
}); err != nil {
{
t.Fatal(err)
}
}
if err := checkColumn(d, cfUnspentTxs, []keyPair{
keyPair{
"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840",
addressToPubKeyHexWithLenght("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00",
},
keyPair{
"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25",
addressToPubKeyHexWithLenght("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "02",
},
keyPair{
"3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71",
addressToPubKeyHexWithLenght("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "00" + addressToPubKeyHexWithLenght("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "02",
},
}); err != nil {
{
t.Fatal(err)
}
}
}