Store data in addresses column in more compact way
Martin Boehm
parent
4e040cb1f0
commit
2552a429e8
|
|
@ -51,25 +51,29 @@ func (w *Worker) getAddressesFromVout(vout *bchain.Vout) (bchain.AddressDescript
|
|||
// setSpendingTxToVout is helper function, that finds transaction that spent given output and sets it to the output
|
||||
// there is no direct index for the operation, it must be found using addresses -> txaddresses -> tx
|
||||
func (w *Worker) setSpendingTxToVout(vout *Vout, txid string, height uint32) error {
|
||||
err := w.db.GetAddrDescTransactions(vout.AddrDesc, height, ^uint32(0), func(t string, index int32, isOutput bool) error {
|
||||
if isOutput == false {
|
||||
tsp, err := w.db.GetTxAddresses(t)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if tsp == nil {
|
||||
glog.Warning("DB inconsistency: tx ", t, ": not found in txAddresses")
|
||||
} else if len(tsp.Inputs) > int(index) {
|
||||
if tsp.Inputs[index].ValueSat.Cmp((*big.Int)(vout.ValueSat)) == 0 {
|
||||
spentTx, spentHeight, err := w.txCache.GetTransaction(t)
|
||||
if err != nil {
|
||||
glog.Warning("Tx ", t, ": not found")
|
||||
} else {
|
||||
if len(spentTx.Vin) > int(index) {
|
||||
if spentTx.Vin[index].Txid == txid {
|
||||
vout.SpentTxID = t
|
||||
vout.SpentHeight = int(spentHeight)
|
||||
vout.SpentIndex = int(index)
|
||||
return &db.StopIteration{}
|
||||
err := w.db.GetAddrDescTransactions(vout.AddrDesc, height, ^uint32(0), func(t string, height uint32, indexes []int32) error {
|
||||
for _, index := range indexes {
|
||||
// take only inputs
|
||||
if index < 0 {
|
||||
index = ^index
|
||||
tsp, err := w.db.GetTxAddresses(t)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if tsp == nil {
|
||||
glog.Warning("DB inconsistency: tx ", t, ": not found in txAddresses")
|
||||
} else if len(tsp.Inputs) > int(index) {
|
||||
if tsp.Inputs[index].ValueSat.Cmp((*big.Int)(vout.ValueSat)) == 0 {
|
||||
spentTx, spentHeight, err := w.txCache.GetTransaction(t)
|
||||
if err != nil {
|
||||
glog.Warning("Tx ", t, ": not found")
|
||||
} else {
|
||||
if len(spentTx.Vin) > int(index) {
|
||||
if spentTx.Vin[index].Txid == txid {
|
||||
vout.SpentTxID = t
|
||||
vout.SpentHeight = int(spentHeight)
|
||||
vout.SpentIndex = int(index)
|
||||
return &db.StopIteration{}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -299,35 +303,50 @@ func (w *Worker) GetTransactionFromBchainTx(bchainTx *bchain.Tx, height uint32,
|
|||
func (w *Worker) getAddressTxids(addrDesc bchain.AddressDescriptor, mempool bool, filter *AddressFilter) ([]string, error) {
|
||||
var err error
|
||||
txids := make([]string, 0, 4)
|
||||
addFilteredTxid := func(txid string, vout int32, isOutput bool) error {
|
||||
if filter.Vout == AddressFilterVoutOff ||
|
||||
(filter.Vout == AddressFilterVoutInputs && !isOutput) ||
|
||||
(filter.Vout == AddressFilterVoutOutputs && isOutput) ||
|
||||
(vout == int32(filter.Vout)) {
|
||||
var callback db.GetTransactionsCallback
|
||||
if filter.Vout == AddressFilterVoutOff {
|
||||
callback = func(txid string, height uint32, indexes []int32) error {
|
||||
txids = append(txids, txid)
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
callback = func(txid string, height uint32, indexes []int32) error {
|
||||
for _, index := range indexes {
|
||||
vout := index
|
||||
if vout < 0 {
|
||||
vout = ^vout
|
||||
}
|
||||
if (filter.Vout == AddressFilterVoutInputs && index < 0) ||
|
||||
(filter.Vout == AddressFilterVoutOutputs && index >= 0) ||
|
||||
(vout == int32(filter.Vout)) {
|
||||
txids = append(txids, txid)
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if mempool {
|
||||
uniqueTxs := make(map[string]struct{})
|
||||
o, err := w.chain.GetMempoolTransactionsForAddrDesc(addrDesc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, m := range o {
|
||||
vout := m.Vout
|
||||
isOutput := true
|
||||
if vout < 0 {
|
||||
isOutput = false
|
||||
vout = ^vout
|
||||
if _, found := uniqueTxs[m.Txid]; !found {
|
||||
l := len(txids)
|
||||
callback(m.Txid, 0, []int32{m.Vout})
|
||||
if len(txids) > l {
|
||||
uniqueTxs[m.Txid] = struct{}{}
|
||||
}
|
||||
}
|
||||
addFilteredTxid(m.Txid, vout, isOutput)
|
||||
}
|
||||
} else {
|
||||
to := filter.ToHeight
|
||||
if to == 0 {
|
||||
to = ^uint32(0)
|
||||
}
|
||||
err = w.db.GetAddrDescTransactions(addrDesc, filter.FromHeight, to, addFilteredTxid)
|
||||
err = w.db.GetAddrDescTransactions(addrDesc, filter.FromHeight, to, callback)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -454,7 +473,7 @@ func (w *Worker) getEthereumTypeAddressBalances(addrDesc bchain.AddressDescripto
|
|||
}
|
||||
if ca != nil {
|
||||
ba = &db.AddrBalance{
|
||||
Txs: uint32(ca.EthTxs),
|
||||
Txs: uint32(ca.TotalTxs),
|
||||
}
|
||||
var b *big.Int
|
||||
b, err = w.chain.EthereumTypeGetBalance(addrDesc)
|
||||
|
|
@ -578,7 +597,6 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, option GetA
|
|||
if err != nil {
|
||||
return nil, errors.Annotatef(err, "getAddressTxids %v true", addrDesc)
|
||||
}
|
||||
txm = GetUniqueTxids(txm)
|
||||
// if there are only unconfirmed transactions, there is no paging
|
||||
if ba == nil {
|
||||
ba = &db.AddrBalance{}
|
||||
|
|
@ -589,7 +607,6 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, option GetA
|
|||
if err != nil {
|
||||
return nil, errors.Annotatef(err, "getAddressTxids %v false", addrDesc)
|
||||
}
|
||||
txc = GetUniqueTxids(txc)
|
||||
bestheight, _, err := w.db.GetBestBlock()
|
||||
if err != nil {
|
||||
return nil, errors.Annotatef(err, "GetBestBlock")
|
||||
|
|
@ -700,7 +717,6 @@ func (w *Worker) GetAddressUtxo(address string, onlyConfirmed bool) ([]AddressUt
|
|||
if err != nil {
|
||||
return nil, errors.Annotatef(err, "getAddressTxids %v true", address)
|
||||
}
|
||||
txm = GetUniqueTxids(txm)
|
||||
mc := make([]*bchain.Tx, len(txm))
|
||||
for i, txid := range txm {
|
||||
// get mempool txs and process their inputs to detect spends between mempool txs
|
||||
|
|
@ -746,9 +762,12 @@ func (w *Worker) GetAddressUtxo(address string, onlyConfirmed bool) ([]AddressUt
|
|||
// ba can be nil if the address is only in mempool!
|
||||
if ba != nil && ba.BalanceSat.Uint64() > 0 {
|
||||
outpoints := make([]bchain.Outpoint, 0, 8)
|
||||
err = w.db.GetAddrDescTransactions(addrDesc, 0, ^uint32(0), func(txid string, vout int32, isOutput bool) error {
|
||||
if isOutput {
|
||||
outpoints = append(outpoints, bchain.Outpoint{Txid: txid, Vout: vout})
|
||||
err = w.db.GetAddrDescTransactions(addrDesc, 0, ^uint32(0), func(txid string, height uint32, indexes []int32) error {
|
||||
for _, index := range indexes {
|
||||
// take only outputs
|
||||
if index >= 0 {
|
||||
outpoints = append(outpoints, bchain.Outpoint{Txid: txid, Vout: index})
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
|
|
|||
10
blockbook.go
10
blockbook.go
|
|
@ -44,8 +44,6 @@ var (
|
|||
blockUntil = flag.Int("blockuntil", -1, "height of the final block")
|
||||
rollbackHeight = flag.Int("rollback", -1, "rollback to the given height and quit")
|
||||
|
||||
queryAddress = flag.String("address", "", "query contents of this address")
|
||||
|
||||
synchronize = flag.Bool("sync", false, "synchronizes until tip, if together with zeromq, keeps index synchronized")
|
||||
repair = flag.Bool("repair", false, "repair the database")
|
||||
prof = flag.String("prof", "", "http server binding [address]:port of the interface to profiling data /debug/pprof/ (default no profiling)")
|
||||
|
|
@ -319,14 +317,8 @@ func main() {
|
|||
}
|
||||
height := uint32(*blockFrom)
|
||||
until := uint32(*blockUntil)
|
||||
address := *queryAddress
|
||||
|
||||
if address != "" {
|
||||
if err = index.GetTransactions(address, height, until, printResult); err != nil {
|
||||
glog.Error("GetTransactions ", err)
|
||||
return
|
||||
}
|
||||
} else if !*synchronize {
|
||||
if !*synchronize {
|
||||
if err = syncWorker.ConnectBlocksParallel(height, until); err != nil {
|
||||
glog.Error("connectBlocksParallel ", err)
|
||||
return
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ import (
|
|||
|
||||
type bulkAddresses struct {
|
||||
bi BlockInfo
|
||||
addresses map[string][]outpoint
|
||||
addresses addressesMap
|
||||
}
|
||||
|
||||
// BulkConnect is used to connect blocks in bulk, faster but if interrupted inconsistent way
|
||||
|
|
@ -40,17 +40,21 @@ const (
|
|||
|
||||
// InitBulkConnect initializes bulk connect and switches DB to inconsistent state
|
||||
func (d *RocksDB) InitBulkConnect() (*BulkConnect, error) {
|
||||
bc := &BulkConnect{
|
||||
b := &BulkConnect{
|
||||
d: d,
|
||||
chainType: d.chainParser.GetChainType(),
|
||||
txAddressesMap: make(map[string]*TxAddresses),
|
||||
balances: make(map[string]*AddrBalance),
|
||||
}
|
||||
if err := d.SetInconsistentState(true); err != nil {
|
||||
return nil, err
|
||||
if b.chainType == bchain.ChainBitcoinType {
|
||||
if err := d.SetInconsistentState(true); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
glog.Info("rocksdb: bulk connect init, db set to inconsistent state")
|
||||
} else {
|
||||
glog.Info("rocksdb: bulk connect init")
|
||||
}
|
||||
glog.Info("rocksdb: bulk connect init, db set to inconsistent state")
|
||||
return bc, nil
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (b *BulkConnect) storeTxAddresses(wb *gorocksdb.WriteBatch, all bool) (int, int, error) {
|
||||
|
|
@ -172,7 +176,7 @@ func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) erro
|
|||
if b.chainType != bchain.ChainBitcoinType {
|
||||
return b.d.ConnectBlock(block)
|
||||
}
|
||||
addresses := make(map[string][]outpoint)
|
||||
addresses := make(addressesMap)
|
||||
if err := b.d.processAddressesBitcoinType(block, addresses, b.txAddressesMap, b.balances); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
161
db/rocksdb.go
161
db/rocksdb.go
|
|
@ -208,9 +208,13 @@ func (e *StopIteration) Error() string {
|
|||
return ""
|
||||
}
|
||||
|
||||
// GetTransactionsCallback is called by GetTransactions/GetAddrDescTransactions for each found tx
|
||||
// indexes contain array of indexes (input negative, output positive) in tx where is given address
|
||||
type GetTransactionsCallback func(txid string, height uint32, indexes []int32) error
|
||||
|
||||
// GetTransactions finds all input/output transactions for address
|
||||
// Transaction are passed to callback function.
|
||||
func (d *RocksDB) GetTransactions(address string, lower uint32, higher uint32, fn func(txid string, vout int32, isOutput bool) error) (err error) {
|
||||
func (d *RocksDB) GetTransactions(address string, lower uint32, higher uint32, fn GetTransactionsCallback) (err error) {
|
||||
if glog.V(1) {
|
||||
glog.Infof("rocksdb: address get %s %d-%d ", address, lower, higher)
|
||||
}
|
||||
|
|
@ -223,47 +227,54 @@ func (d *RocksDB) GetTransactions(address string, lower uint32, higher uint32, f
|
|||
|
||||
// GetAddrDescTransactions finds all input/output transactions for address descriptor
|
||||
// Transaction are passed to callback function in the order from newest block to the oldest
|
||||
func (d *RocksDB) GetAddrDescTransactions(addrDesc bchain.AddressDescriptor, lower uint32, higher uint32, fn func(txid string, vout int32, isOutput bool) error) (err error) {
|
||||
kstart := packAddressKey(addrDesc, higher)
|
||||
kstop := packAddressKey(addrDesc, lower)
|
||||
|
||||
func (d *RocksDB) GetAddrDescTransactions(addrDesc bchain.AddressDescriptor, lower uint32, higher uint32, fn GetTransactionsCallback) (err error) {
|
||||
txidUnpackedLen := d.chainParser.PackedTxidLen()
|
||||
startKey := packAddressKey(addrDesc, higher)
|
||||
stopKey := packAddressKey(addrDesc, lower)
|
||||
indexes := make([]int32, 0, 16)
|
||||
it := d.db.NewIteratorCF(d.ro, d.cfh[cfAddresses])
|
||||
defer it.Close()
|
||||
|
||||
for it.Seek(kstart); it.Valid(); it.Next() {
|
||||
for it.Seek(startKey); it.Valid(); it.Next() {
|
||||
key := it.Key().Data()
|
||||
val := it.Value().Data()
|
||||
if bytes.Compare(key, kstop) > 0 {
|
||||
if bytes.Compare(key, stopKey) > 0 {
|
||||
break
|
||||
}
|
||||
outpoints, err := d.unpackOutpoints(val)
|
||||
if glog.V(2) {
|
||||
glog.Infof("rocksdb: addresses %s: %s", hex.EncodeToString(key), hex.EncodeToString(val))
|
||||
}
|
||||
_, height, err := unpackAddressKey(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if glog.V(2) {
|
||||
glog.Infof("rocksdb: output %s: %s", hex.EncodeToString(key), hex.EncodeToString(val))
|
||||
}
|
||||
for _, o := range outpoints {
|
||||
var vout int32
|
||||
var isOutput bool
|
||||
if o.index < 0 {
|
||||
vout = int32(^o.index)
|
||||
isOutput = false
|
||||
} else {
|
||||
vout = int32(o.index)
|
||||
isOutput = true
|
||||
}
|
||||
tx, err := d.chainParser.UnpackTxid(o.btxID)
|
||||
for len(val) > txidUnpackedLen {
|
||||
tx, err := d.chainParser.UnpackTxid(val[:txidUnpackedLen])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := fn(tx, vout, isOutput); err != nil {
|
||||
indexes = indexes[:0]
|
||||
val = val[txidUnpackedLen:]
|
||||
for {
|
||||
index, l := unpackVarint32(val)
|
||||
indexes = append(indexes, index>>1)
|
||||
val = val[l:]
|
||||
if index&1 == 1 {
|
||||
break
|
||||
} else if len(val) == 0 {
|
||||
glog.Warningf("rocksdb: addresses contain incorrect data %s: %s", hex.EncodeToString(key), hex.EncodeToString(val))
|
||||
break
|
||||
}
|
||||
}
|
||||
if err := fn(tx, height, indexes); err != nil {
|
||||
if _, ok := err.(*StopIteration); ok {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(val) != 0 {
|
||||
glog.Warningf("rocksdb: addresses contain incorrect data %s: %s", hex.EncodeToString(key), hex.EncodeToString(val))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -287,7 +298,7 @@ func (d *RocksDB) ConnectBlock(block *bchain.Block) error {
|
|||
if err := d.writeHeightFromBlock(wb, block, opInsert); err != nil {
|
||||
return err
|
||||
}
|
||||
addresses := make(map[string][]outpoint)
|
||||
addresses := make(addressesMap)
|
||||
if chainType == bchain.ChainBitcoinType {
|
||||
txAddressesMap := make(map[string]*TxAddresses)
|
||||
balances := make(map[string]*AddrBalance)
|
||||
|
|
@ -327,6 +338,16 @@ func (d *RocksDB) ConnectBlock(block *bchain.Block) error {
|
|||
|
||||
// Addresses index
|
||||
|
||||
type txIndexes struct {
|
||||
btxID []byte
|
||||
indexes []int32
|
||||
}
|
||||
|
||||
// addressesMap is a map of addresses in a block
|
||||
// each address contains a slice of transactions with indexes where the address appears
|
||||
// slice is used instead of map so that order is defined and also search in case of few items
|
||||
type addressesMap map[string][]txIndexes
|
||||
|
||||
type outpoint struct {
|
||||
btxID []byte
|
||||
index int32
|
||||
|
|
@ -391,7 +412,7 @@ func (d *RocksDB) GetAndResetConnectBlockStats() string {
|
|||
return s
|
||||
}
|
||||
|
||||
func (d *RocksDB) processAddressesBitcoinType(block *bchain.Block, addresses map[string][]outpoint, txAddressesMap map[string]*TxAddresses, balances map[string]*AddrBalance) error {
|
||||
func (d *RocksDB) processAddressesBitcoinType(block *bchain.Block, addresses addressesMap, txAddressesMap map[string]*TxAddresses, balances map[string]*AddrBalance) error {
|
||||
blockTxIDs := make([][]byte, len(block.Txs))
|
||||
blockTxAddresses := make([]*TxAddresses, len(block.Txs))
|
||||
// first process all outputs so that inputs can point to txs in this block
|
||||
|
|
@ -423,16 +444,6 @@ func (d *RocksDB) processAddressesBitcoinType(block *bchain.Block, addresses map
|
|||
}
|
||||
tao.AddrDesc = addrDesc
|
||||
strAddrDesc := string(addrDesc)
|
||||
// check that the address was used already in this block
|
||||
o, processed := addresses[strAddrDesc]
|
||||
if processed {
|
||||
// check that the address was already used in this tx
|
||||
processed = processedInTx(o, btxID)
|
||||
}
|
||||
addresses[strAddrDesc] = append(o, outpoint{
|
||||
btxID: btxID,
|
||||
index: int32(i),
|
||||
})
|
||||
ab, e := balances[strAddrDesc]
|
||||
if !e {
|
||||
ab, err = d.GetAddrDescBalance(addrDesc)
|
||||
|
|
@ -447,11 +458,11 @@ func (d *RocksDB) processAddressesBitcoinType(block *bchain.Block, addresses map
|
|||
} else {
|
||||
d.cbs.balancesHit++
|
||||
}
|
||||
// add number of trx in balance only once, address can be multiple times in tx
|
||||
if !processed {
|
||||
ab.BalanceSat.Add(&ab.BalanceSat, &output.ValueSat)
|
||||
counted := addToAddressesMap(addresses, strAddrDesc, btxID, int32(i))
|
||||
if !counted {
|
||||
ab.Txs++
|
||||
}
|
||||
ab.BalanceSat.Add(&ab.BalanceSat, &output.ValueSat)
|
||||
}
|
||||
}
|
||||
// process inputs
|
||||
|
|
@ -507,16 +518,6 @@ func (d *RocksDB) processAddressesBitcoinType(block *bchain.Block, addresses map
|
|||
continue
|
||||
}
|
||||
strAddrDesc := string(ot.AddrDesc)
|
||||
// check that the address was used already in this block
|
||||
o, processed := addresses[strAddrDesc]
|
||||
if processed {
|
||||
// check that the address was already used in this tx
|
||||
processed = processedInTx(o, spendingTxid)
|
||||
}
|
||||
addresses[strAddrDesc] = append(o, outpoint{
|
||||
btxID: spendingTxid,
|
||||
index: ^int32(i),
|
||||
})
|
||||
ab, e := balances[strAddrDesc]
|
||||
if !e {
|
||||
ab, err = d.GetAddrDescBalance(ot.AddrDesc)
|
||||
|
|
@ -531,8 +532,8 @@ func (d *RocksDB) processAddressesBitcoinType(block *bchain.Block, addresses map
|
|||
} else {
|
||||
d.cbs.balancesHit++
|
||||
}
|
||||
// add number of trx in balance only once, address can be multiple times in tx
|
||||
if !processed {
|
||||
counted := addToAddressesMap(addresses, strAddrDesc, spendingTxid, ^int32(i))
|
||||
if !counted {
|
||||
ab.Txs++
|
||||
}
|
||||
ab.BalanceSat.Sub(&ab.BalanceSat, &ot.ValueSat)
|
||||
|
|
@ -545,20 +546,31 @@ func (d *RocksDB) processAddressesBitcoinType(block *bchain.Block, addresses map
|
|||
return nil
|
||||
}
|
||||
|
||||
func processedInTx(o []outpoint, btxID []byte) bool {
|
||||
for _, op := range o {
|
||||
if bytes.Equal(btxID, op.btxID) {
|
||||
return true
|
||||
func addToAddressesMap(addresses addressesMap, strAddrDesc string, btxID []byte, index int32) bool {
|
||||
// check that the address was used already in this block
|
||||
// if not found, it has certainly not been counted
|
||||
at, found := addresses[strAddrDesc]
|
||||
if found {
|
||||
// check that the address was already used in this tx
|
||||
for i, t := range at {
|
||||
if bytes.Equal(btxID, t.btxID) {
|
||||
at[i].indexes = append(at[i].indexes, index)
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
addresses[strAddrDesc] = append(at, txIndexes{
|
||||
btxID: btxID,
|
||||
indexes: []int32{index},
|
||||
})
|
||||
return false
|
||||
}
|
||||
|
||||
func (d *RocksDB) storeAddresses(wb *gorocksdb.WriteBatch, height uint32, addresses map[string][]outpoint) error {
|
||||
for addrDesc, outpoints := range addresses {
|
||||
func (d *RocksDB) storeAddresses(wb *gorocksdb.WriteBatch, height uint32, addresses addressesMap) error {
|
||||
for addrDesc, txi := range addresses {
|
||||
ba := bchain.AddressDescriptor(addrDesc)
|
||||
key := packAddressKey(ba, height)
|
||||
val := d.packOutpoints(outpoints)
|
||||
val := d.packTxIndexes(txi)
|
||||
wb.PutCF(d.cfh[cfAddresses], key, val)
|
||||
}
|
||||
return nil
|
||||
|
|
@ -813,6 +825,23 @@ func unpackTxOutput(to *TxOutput, buf []byte) int {
|
|||
return l + al
|
||||
}
|
||||
|
||||
func (d *RocksDB) packTxIndexes(txi []txIndexes) []byte {
|
||||
buf := make([]byte, 0, 32)
|
||||
bvout := make([]byte, vlq.MaxLen32)
|
||||
for _, t := range txi {
|
||||
buf = append(buf, []byte(t.btxID)...)
|
||||
for i, index := range t.indexes {
|
||||
index <<= 1
|
||||
if i == len(t.indexes)-1 {
|
||||
index |= 1
|
||||
}
|
||||
l := packVarint32(index, bvout)
|
||||
buf = append(buf, bvout[:l]...)
|
||||
}
|
||||
}
|
||||
return buf
|
||||
}
|
||||
|
||||
func (d *RocksDB) packOutpoints(outpoints []outpoint) []byte {
|
||||
buf := make([]byte, 0, 32)
|
||||
bvout := make([]byte, vlq.MaxLen32)
|
||||
|
|
@ -824,22 +853,6 @@ func (d *RocksDB) packOutpoints(outpoints []outpoint) []byte {
|
|||
return buf
|
||||
}
|
||||
|
||||
func (d *RocksDB) unpackOutpoints(buf []byte) ([]outpoint, error) {
|
||||
txidUnpackedLen := d.chainParser.PackedTxidLen()
|
||||
outpoints := make([]outpoint, 0, 8)
|
||||
for i := 0; i < len(buf); {
|
||||
btxID := append([]byte(nil), buf[i:i+txidUnpackedLen]...)
|
||||
i += txidUnpackedLen
|
||||
vout, voutLen := unpackVarint32(buf[i:])
|
||||
i += voutLen
|
||||
outpoints = append(outpoints, outpoint{
|
||||
btxID: btxID,
|
||||
index: vout,
|
||||
})
|
||||
}
|
||||
return outpoints, nil
|
||||
}
|
||||
|
||||
func (d *RocksDB) unpackNOutpoints(buf []byte) ([]outpoint, int, error) {
|
||||
txidUnpackedLen := d.chainParser.PackedTxidLen()
|
||||
n, p := unpackVaruint(buf)
|
||||
|
|
|
|||
|
|
@ -18,10 +18,11 @@ type AddrContract struct {
|
|||
Txs uint
|
||||
}
|
||||
|
||||
// AddrContracts is array of contracts with
|
||||
// AddrContracts contains number of transactions and contracts for an address
|
||||
type AddrContracts struct {
|
||||
EthTxs uint
|
||||
Contracts []AddrContract
|
||||
TotalTxs uint
|
||||
NonContractTxs uint
|
||||
Contracts []AddrContract
|
||||
}
|
||||
|
||||
func (d *RocksDB) storeAddressContracts(wb *gorocksdb.WriteBatch, acm map[string]*AddrContracts) error {
|
||||
|
|
@ -29,11 +30,13 @@ func (d *RocksDB) storeAddressContracts(wb *gorocksdb.WriteBatch, acm map[string
|
|||
varBuf := make([]byte, vlq.MaxLen64)
|
||||
for addrDesc, acs := range acm {
|
||||
// address with 0 contracts is removed from db - happens on disconnect
|
||||
if acs == nil || (acs.EthTxs == 0 && len(acs.Contracts) == 0) {
|
||||
if acs == nil || (acs.NonContractTxs == 0 && len(acs.Contracts) == 0) {
|
||||
wb.DeleteCF(d.cfh[cfAddressContracts], bchain.AddressDescriptor(addrDesc))
|
||||
} else {
|
||||
buf = buf[:0]
|
||||
l := packVaruint(acs.EthTxs, varBuf)
|
||||
l := packVaruint(acs.TotalTxs, varBuf)
|
||||
buf = append(buf, varBuf[:l]...)
|
||||
l = packVaruint(acs.NonContractTxs, varBuf)
|
||||
buf = append(buf, varBuf[:l]...)
|
||||
for _, ac := range acs.Contracts {
|
||||
buf = append(buf, ac.Contract...)
|
||||
|
|
@ -57,9 +60,11 @@ func (d *RocksDB) GetAddrDescContracts(addrDesc bchain.AddressDescriptor) (*Addr
|
|||
if len(buf) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
c := make([]AddrContract, 0, 4)
|
||||
et, l := unpackVaruint(buf)
|
||||
tt, l := unpackVaruint(buf)
|
||||
buf = buf[l:]
|
||||
nct, l := unpackVaruint(buf)
|
||||
buf = buf[l:]
|
||||
c := make([]AddrContract, 0, 4)
|
||||
for len(buf) > 0 {
|
||||
if len(buf) < eth.EthereumTypeAddressDescriptorLen {
|
||||
return nil, errors.New("Invalid data stored in cfAddressContracts for AddrDesc " + addrDesc.String())
|
||||
|
|
@ -72,7 +77,11 @@ func (d *RocksDB) GetAddrDescContracts(addrDesc bchain.AddressDescriptor) (*Addr
|
|||
})
|
||||
buf = buf[eth.EthereumTypeAddressDescriptorLen+l:]
|
||||
}
|
||||
return &AddrContracts{EthTxs: et, Contracts: c}, nil
|
||||
return &AddrContracts{
|
||||
TotalTxs: tt,
|
||||
NonContractTxs: nct,
|
||||
Contracts: c,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func findContractInAddressContracts(contract bchain.AddressDescriptor, contracts []AddrContract) (int, bool) {
|
||||
|
|
@ -84,7 +93,7 @@ func findContractInAddressContracts(contract bchain.AddressDescriptor, contracts
|
|||
return 0, false
|
||||
}
|
||||
|
||||
func (d *RocksDB) addToAddressesAndContractsEthereumType(addrDesc bchain.AddressDescriptor, btxID []byte, index int32, contract bchain.AddressDescriptor, addresses map[string][]outpoint, addressContracts map[string]*AddrContracts) error {
|
||||
func (d *RocksDB) addToAddressesAndContractsEthereumType(addrDesc bchain.AddressDescriptor, btxID []byte, index int32, contract bchain.AddressDescriptor, addresses addressesMap, addressContracts map[string]*AddrContracts, addTxCount bool) error {
|
||||
var err error
|
||||
strAddrDesc := string(addrDesc)
|
||||
ac, e := addressContracts[strAddrDesc]
|
||||
|
|
@ -102,7 +111,9 @@ func (d *RocksDB) addToAddressesAndContractsEthereumType(addrDesc bchain.Address
|
|||
d.cbs.balancesHit++
|
||||
}
|
||||
if contract == nil {
|
||||
ac.EthTxs++
|
||||
if addTxCount {
|
||||
ac.NonContractTxs++
|
||||
}
|
||||
} else {
|
||||
// locate the contract and set i to the index in the array of contracts
|
||||
i, found := findContractInAddressContracts(contract, ac.Contracts)
|
||||
|
|
@ -116,12 +127,14 @@ func (d *RocksDB) addToAddressesAndContractsEthereumType(addrDesc bchain.Address
|
|||
} else {
|
||||
index = int32(i + 1)
|
||||
}
|
||||
ac.Contracts[i].Txs++
|
||||
if addTxCount {
|
||||
ac.Contracts[i].Txs++
|
||||
}
|
||||
}
|
||||
counted := addToAddressesMap(addresses, strAddrDesc, btxID, index)
|
||||
if !counted {
|
||||
ac.TotalTxs++
|
||||
}
|
||||
addresses[strAddrDesc] = append(addresses[strAddrDesc], outpoint{
|
||||
btxID: btxID,
|
||||
index: index,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -135,7 +148,7 @@ type ethBlockTx struct {
|
|||
contracts []ethBlockTxContract
|
||||
}
|
||||
|
||||
func (d *RocksDB) processAddressesEthereumType(block *bchain.Block, addresses map[string][]outpoint, addressContracts map[string]*AddrContracts) ([]ethBlockTx, error) {
|
||||
func (d *RocksDB) processAddressesEthereumType(block *bchain.Block, addresses addressesMap, addressContracts map[string]*AddrContracts) ([]ethBlockTx, error) {
|
||||
blockTxs := make([]ethBlockTx, len(block.Txs))
|
||||
for txi, tx := range block.Txs {
|
||||
btxID, err := d.chainParser.PackTxid(tx.Txid)
|
||||
|
|
@ -144,9 +157,10 @@ func (d *RocksDB) processAddressesEthereumType(block *bchain.Block, addresses ma
|
|||
}
|
||||
blockTx := &blockTxs[txi]
|
||||
blockTx.btxID = btxID
|
||||
var from, to bchain.AddressDescriptor
|
||||
// there is only one output address in EthereumType transaction, store it in format txid 0
|
||||
if len(tx.Vout) == 1 && len(tx.Vout[0].ScriptPubKey.Addresses) == 1 {
|
||||
addrDesc, err := d.chainParser.GetAddrDescFromAddress(tx.Vout[0].ScriptPubKey.Addresses[0])
|
||||
to, err = d.chainParser.GetAddrDescFromAddress(tx.Vout[0].ScriptPubKey.Addresses[0])
|
||||
if err != nil {
|
||||
// do not log ErrAddressMissing, transactions can be without to address (for example eth contracts)
|
||||
if err != bchain.ErrAddressMissing {
|
||||
|
|
@ -154,24 +168,24 @@ func (d *RocksDB) processAddressesEthereumType(block *bchain.Block, addresses ma
|
|||
}
|
||||
continue
|
||||
}
|
||||
if err = d.addToAddressesAndContractsEthereumType(addrDesc, btxID, 0, nil, addresses, addressContracts); err != nil {
|
||||
if err = d.addToAddressesAndContractsEthereumType(to, btxID, 0, nil, addresses, addressContracts, true); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
blockTx.to = addrDesc
|
||||
blockTx.to = to
|
||||
}
|
||||
// there is only one input address in EthereumType transaction, store it in format txid ^0
|
||||
if len(tx.Vin) == 1 && len(tx.Vin[0].Addresses) == 1 {
|
||||
addrDesc, err := d.chainParser.GetAddrDescFromAddress(tx.Vin[0].Addresses[0])
|
||||
from, err = d.chainParser.GetAddrDescFromAddress(tx.Vin[0].Addresses[0])
|
||||
if err != nil {
|
||||
if err != bchain.ErrAddressMissing {
|
||||
glog.Warningf("rocksdb: addrDesc: %v - height %d, tx %v, input", err, block.Height, tx.Txid)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err = d.addToAddressesAndContractsEthereumType(addrDesc, btxID, ^int32(0), nil, addresses, addressContracts); err != nil {
|
||||
if err = d.addToAddressesAndContractsEthereumType(from, btxID, ^int32(0), nil, addresses, addressContracts, !bytes.Equal(from, to)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
blockTx.from = addrDesc
|
||||
blockTx.from = from
|
||||
}
|
||||
// store erc20 transfers
|
||||
erc20, err := d.chainParser.EthereumTypeGetErc20FromTx(&tx)
|
||||
|
|
@ -179,6 +193,7 @@ func (d *RocksDB) processAddressesEthereumType(block *bchain.Block, addresses ma
|
|||
glog.Warningf("rocksdb: GetErc20FromTx %v - height %d, tx %v", err, block.Height, tx.Txid)
|
||||
}
|
||||
blockTx.contracts = make([]ethBlockTxContract, len(erc20)*2)
|
||||
j := 0
|
||||
for i, t := range erc20 {
|
||||
var contract, from, to bchain.AddressDescriptor
|
||||
contract, err = d.chainParser.GetAddrDescFromAddress(t.Contract)
|
||||
|
|
@ -192,19 +207,26 @@ func (d *RocksDB) processAddressesEthereumType(block *bchain.Block, addresses ma
|
|||
glog.Warningf("rocksdb: GetErc20FromTx %v - height %d, tx %v, transfer %v", err, block.Height, tx.Txid, t)
|
||||
continue
|
||||
}
|
||||
if err = d.addToAddressesAndContractsEthereumType(from, btxID, ^int32(i), contract, addresses, addressContracts); err != nil {
|
||||
if err = d.addToAddressesAndContractsEthereumType(from, btxID, ^int32(i), contract, addresses, addressContracts, true); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bc := &blockTx.contracts[i*2]
|
||||
eq := bytes.Equal(from, to)
|
||||
bc := &blockTx.contracts[j]
|
||||
j++
|
||||
bc.addr = from
|
||||
bc.contract = contract
|
||||
if err = d.addToAddressesAndContractsEthereumType(to, btxID, int32(i), contract, addresses, addressContracts); err != nil {
|
||||
if err = d.addToAddressesAndContractsEthereumType(to, btxID, int32(i), contract, addresses, addressContracts, !eq); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bc = &blockTx.contracts[i*2+1]
|
||||
bc.addr = to
|
||||
bc.contract = contract
|
||||
// add to address to blockTx.contracts only if it is different from from address
|
||||
if !eq {
|
||||
bc = &blockTx.contracts[j]
|
||||
j++
|
||||
bc.addr = to
|
||||
bc.contract = contract
|
||||
}
|
||||
}
|
||||
blockTx.contracts = blockTx.contracts[:j]
|
||||
}
|
||||
return blockTxs, nil
|
||||
}
|
||||
|
|
@ -308,7 +330,7 @@ func (d *RocksDB) getBlockTxsEthereumType(height uint32) ([]ethBlockTx, error) {
|
|||
|
||||
func (d *RocksDB) disconnectBlockTxsEthereumType(wb *gorocksdb.WriteBatch, height uint32, blockTxs []ethBlockTx, contracts map[string]*AddrContracts) error {
|
||||
glog.Info("Disconnecting block ", height, " containing ", len(blockTxs), " transactions")
|
||||
addresses := make(map[string]struct{})
|
||||
addresses := make(map[string]map[string]struct{})
|
||||
disconnectAddress := func(btxID []byte, addrDesc, contract bchain.AddressDescriptor) error {
|
||||
var err error
|
||||
// do not process empty address
|
||||
|
|
@ -316,7 +338,19 @@ func (d *RocksDB) disconnectBlockTxsEthereumType(wb *gorocksdb.WriteBatch, heigh
|
|||
return nil
|
||||
}
|
||||
s := string(addrDesc)
|
||||
addresses[s] = struct{}{}
|
||||
txid := string(btxID)
|
||||
// find if tx for this address was already encountered
|
||||
mtx, ftx := addresses[s]
|
||||
if !ftx {
|
||||
mtx = make(map[string]struct{})
|
||||
mtx[txid] = struct{}{}
|
||||
addresses[s] = mtx
|
||||
} else {
|
||||
_, ftx = mtx[txid]
|
||||
if !ftx {
|
||||
mtx[txid] = struct{}{}
|
||||
}
|
||||
}
|
||||
c, fc := contracts[s]
|
||||
if !fc {
|
||||
c, err = d.GetAddrDescContracts(addrDesc)
|
||||
|
|
@ -326,9 +360,12 @@ func (d *RocksDB) disconnectBlockTxsEthereumType(wb *gorocksdb.WriteBatch, heigh
|
|||
contracts[s] = c
|
||||
}
|
||||
if c != nil {
|
||||
if !ftx {
|
||||
c.TotalTxs--
|
||||
}
|
||||
if contract == nil {
|
||||
if c.EthTxs > 0 {
|
||||
c.EthTxs--
|
||||
if c.NonContractTxs > 0 {
|
||||
c.NonContractTxs--
|
||||
} else {
|
||||
glog.Warning("AddressContracts ", addrDesc, ", EthTxs would be negative, tx ", hex.EncodeToString(btxID))
|
||||
}
|
||||
|
|
@ -357,8 +394,11 @@ func (d *RocksDB) disconnectBlockTxsEthereumType(wb *gorocksdb.WriteBatch, heigh
|
|||
if err := disconnectAddress(blockTx.btxID, blockTx.from, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := disconnectAddress(blockTx.btxID, blockTx.to, nil); err != nil {
|
||||
return err
|
||||
// if from==to, tx is counted only once and does not have to be disconnected again
|
||||
if !bytes.Equal(blockTx.from, blockTx.to) {
|
||||
if err := disconnectAddress(blockTx.btxID, blockTx.to, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, c := range blockTx.contracts {
|
||||
if err := disconnectAddress(blockTx.btxID, c.addr, c.contract); err != nil {
|
||||
|
|
|
|||
|
|
@ -32,12 +32,11 @@ func verifyAfterEthereumTypeBlock1(t *testing.T, d *RocksDB, afterDisconnect boo
|
|||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// the vout is encoded as signed varint, i.e. value * 2 for positive values, abs(value)*2 + 1 for negative values
|
||||
if err := checkColumn(d, cfAddresses, []keyPair{
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddr3e, 4321000, d), dbtestdata.EthTxidB1T1 + "01", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddr55, 4321000, d), dbtestdata.EthTxidB1T1 + "00" + dbtestdata.EthTxidB1T2 + "02", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddr20, 4321000, d), dbtestdata.EthTxidB1T2 + "01" + dbtestdata.EthTxidB1T2 + "03", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddrContract4a, 4321000, d), dbtestdata.EthTxidB1T2 + "00", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddr3e, 4321000, d), txIndexesHex(dbtestdata.EthTxidB1T1, []int32{^0}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddr55, 4321000, d), txIndexesHex(dbtestdata.EthTxidB1T1, []int32{0}) + txIndexesHex(dbtestdata.EthTxidB1T2, []int32{1}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddr20, 4321000, d), txIndexesHex(dbtestdata.EthTxidB1T2, []int32{^0, ^1}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddrContract4a, 4321000, d), txIndexesHex(dbtestdata.EthTxidB1T2, []int32{0}), nil},
|
||||
}); err != nil {
|
||||
{
|
||||
t.Fatal(err)
|
||||
|
|
@ -45,10 +44,10 @@ func verifyAfterEthereumTypeBlock1(t *testing.T, d *RocksDB, afterDisconnect boo
|
|||
}
|
||||
|
||||
if err := checkColumn(d, cfAddressContracts, []keyPair{
|
||||
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser), "01", nil},
|
||||
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser), "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01", nil},
|
||||
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser), "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01", nil},
|
||||
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser), "01", nil},
|
||||
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser), "0101", nil},
|
||||
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser), "0201" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01", nil},
|
||||
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser), "0101" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01", nil},
|
||||
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser), "0101", nil},
|
||||
}); err != nil {
|
||||
{
|
||||
t.Fatal(err)
|
||||
|
|
@ -97,17 +96,16 @@ func verifyAfterEthereumTypeBlock2(t *testing.T, d *RocksDB) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// the vout is encoded as signed varint, i.e. value * 2 for positive values, abs(value)*2 + 1 for negative values
|
||||
if err := checkColumn(d, cfAddresses, []keyPair{
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddr3e, 4321000, d), dbtestdata.EthTxidB1T1 + "01", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddr55, 4321000, d), dbtestdata.EthTxidB1T1 + "00" + dbtestdata.EthTxidB1T2 + "02", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddr20, 4321000, d), dbtestdata.EthTxidB1T2 + "01" + dbtestdata.EthTxidB1T2 + "03", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddrContract4a, 4321000, d), dbtestdata.EthTxidB1T2 + "00", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddr55, 4321001, d), dbtestdata.EthTxidB2T1 + "01" + dbtestdata.EthTxidB2T2 + "05" + dbtestdata.EthTxidB2T2 + "02", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddr9f, 4321001, d), dbtestdata.EthTxidB2T1 + "00", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddr4b, 4321001, d), dbtestdata.EthTxidB2T2 + "01" + dbtestdata.EthTxidB2T2 + "02" + dbtestdata.EthTxidB2T2 + "05" + dbtestdata.EthTxidB2T2 + "04" + dbtestdata.EthTxidB2T2 + "03", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddr7b, 4321001, d), dbtestdata.EthTxidB2T2 + "03" + dbtestdata.EthTxidB2T2 + "04", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddrContract47, 4321001, d), dbtestdata.EthTxidB2T2 + "00", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddr3e, 4321000, d), txIndexesHex(dbtestdata.EthTxidB1T1, []int32{^0}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddr55, 4321000, d), txIndexesHex(dbtestdata.EthTxidB1T1, []int32{0}) + txIndexesHex(dbtestdata.EthTxidB1T2, []int32{1}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddr20, 4321000, d), txIndexesHex(dbtestdata.EthTxidB1T2, []int32{^0, ^1}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddrContract4a, 4321000, d), txIndexesHex(dbtestdata.EthTxidB1T2, []int32{0}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddr55, 4321001, d), txIndexesHex(dbtestdata.EthTxidB2T1, []int32{^0}) + txIndexesHex(dbtestdata.EthTxidB2T2, []int32{^2, 1}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddr9f, 4321001, d), txIndexesHex(dbtestdata.EthTxidB2T1, []int32{0}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddr4b, 4321001, d), txIndexesHex(dbtestdata.EthTxidB2T2, []int32{^0, 1, ^2, 2, ^1}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddr7b, 4321001, d), txIndexesHex(dbtestdata.EthTxidB2T2, []int32{^1, 2}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.EthAddrContract47, 4321001, d), txIndexesHex(dbtestdata.EthTxidB2T2, []int32{0}), nil},
|
||||
}); err != nil {
|
||||
{
|
||||
t.Fatal(err)
|
||||
|
|
@ -115,14 +113,14 @@ func verifyAfterEthereumTypeBlock2(t *testing.T, d *RocksDB) {
|
|||
}
|
||||
|
||||
if err := checkColumn(d, cfAddressContracts, []keyPair{
|
||||
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser), "01", nil},
|
||||
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser), "02" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "02" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + "01", nil},
|
||||
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser), "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01", nil},
|
||||
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser), "01", nil},
|
||||
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr9f, d.chainParser), "01", nil},
|
||||
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser), "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + "02" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "02", nil},
|
||||
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr7b, d.chainParser), "00" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + "01", nil},
|
||||
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract47, d.chainParser), "01", nil},
|
||||
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser), "0101", nil},
|
||||
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser), "0402" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "02" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + "01", nil},
|
||||
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser), "0101" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01", nil},
|
||||
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser), "0101", nil},
|
||||
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr9f, d.chainParser), "0101", nil},
|
||||
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser), "0101" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + "02" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "02", nil},
|
||||
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr7b, d.chainParser), "0100" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + "01", nil},
|
||||
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract47, d.chainParser), "0101", nil},
|
||||
}); err != nil {
|
||||
{
|
||||
t.Fatal(err)
|
||||
|
|
@ -184,14 +182,14 @@ func TestRocksDB_Index_EthereumType(t *testing.T) {
|
|||
verifyAfterEthereumTypeBlock2(t, d)
|
||||
|
||||
// get transactions for various addresses / low-high ranges
|
||||
verifyGetTransactions(t, d, "0x"+dbtestdata.EthAddr55, 0, 10000000, []txidVoutOutput{
|
||||
txidVoutOutput{"0x" + dbtestdata.EthTxidB2T1, 0, false},
|
||||
txidVoutOutput{"0x" + dbtestdata.EthTxidB2T2, 2, false},
|
||||
txidVoutOutput{"0x" + dbtestdata.EthTxidB2T2, 1, true},
|
||||
txidVoutOutput{"0x" + dbtestdata.EthTxidB1T1, 0, true},
|
||||
txidVoutOutput{"0x" + dbtestdata.EthTxidB1T2, 1, true},
|
||||
verifyGetTransactions(t, d, "0x"+dbtestdata.EthAddr55, 0, 10000000, []txidIndex{
|
||||
{"0x" + dbtestdata.EthTxidB2T1, ^0},
|
||||
{"0x" + dbtestdata.EthTxidB2T2, ^2},
|
||||
{"0x" + dbtestdata.EthTxidB2T2, 1},
|
||||
{"0x" + dbtestdata.EthTxidB1T1, 0},
|
||||
{"0x" + dbtestdata.EthTxidB1T2, 1},
|
||||
}, nil)
|
||||
verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eBad", 500000, 1000000, []txidVoutOutput{}, errors.New("Address missing"))
|
||||
verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eBad", 500000, 1000000, []txidIndex{}, errors.New("Address missing"))
|
||||
|
||||
// GetBestBlock
|
||||
height, hash, err := d.GetBestBlock()
|
||||
|
|
|
|||
|
|
@ -103,6 +103,19 @@ func addressKeyHex(a string, height uint32, d *RocksDB) string {
|
|||
return dbtestdata.AddressToPubKeyHex(a, d.chainParser) + uintToHex(^height)
|
||||
}
|
||||
|
||||
func txIndexesHex(tx string, indexes []int32) string {
|
||||
buf := make([]byte, vlq.MaxLen32)
|
||||
for i, index := range indexes {
|
||||
index <<= 1
|
||||
if i == len(indexes)-1 {
|
||||
index |= 1
|
||||
}
|
||||
l := packVarint32(index, buf)
|
||||
tx += hex.EncodeToString(buf[:l])
|
||||
}
|
||||
return tx
|
||||
}
|
||||
|
||||
// keyPair is used to compare given key value in DB with expected
|
||||
// for more complicated compares it is possible to specify CompareFunc
|
||||
type keyPair struct {
|
||||
|
|
@ -172,11 +185,11 @@ func verifyAfterBitcoinTypeBlock1(t *testing.T, d *RocksDB, afterDisconnect bool
|
|||
}
|
||||
// the vout is encoded as signed varint, i.e. value * 2 for non negative values
|
||||
if err := checkColumn(d, cfAddresses, []keyPair{
|
||||
keyPair{addressKeyHex(dbtestdata.Addr1, 225493, d), dbtestdata.TxidB1T1 + "00", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr2, 225493, d), dbtestdata.TxidB1T1 + "02", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr3, 225493, d), dbtestdata.TxidB1T2 + "00", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr4, 225493, d), dbtestdata.TxidB1T2 + "02", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr5, 225493, d), dbtestdata.TxidB1T2 + "04", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr1, 225493, d), txIndexesHex(dbtestdata.TxidB1T1, []int32{0}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr2, 225493, d), txIndexesHex(dbtestdata.TxidB1T1, []int32{1}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr3, 225493, d), txIndexesHex(dbtestdata.TxidB1T2, []int32{0}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr4, 225493, d), txIndexesHex(dbtestdata.TxidB1T2, []int32{1}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr5, 225493, d), txIndexesHex(dbtestdata.TxidB1T2, []int32{2}), nil},
|
||||
}); err != nil {
|
||||
{
|
||||
t.Fatal(err)
|
||||
|
|
@ -257,20 +270,20 @@ func verifyAfterBitcoinTypeBlock2(t *testing.T, d *RocksDB) {
|
|||
}
|
||||
}
|
||||
if err := checkColumn(d, cfAddresses, []keyPair{
|
||||
keyPair{addressKeyHex(dbtestdata.Addr1, 225493, d), dbtestdata.TxidB1T1 + "00", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr2, 225493, d), dbtestdata.TxidB1T1 + "02", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr3, 225493, d), dbtestdata.TxidB1T2 + "00", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr4, 225493, d), dbtestdata.TxidB1T2 + "02", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr5, 225493, d), dbtestdata.TxidB1T2 + "04", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr6, 225494, d), dbtestdata.TxidB2T1 + "00" + dbtestdata.TxidB2T2 + "01", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr7, 225494, d), dbtestdata.TxidB2T1 + "02", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr8, 225494, d), dbtestdata.TxidB2T2 + "00", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr9, 225494, d), dbtestdata.TxidB2T2 + "02", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr3, 225494, d), dbtestdata.TxidB2T1 + "01", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr2, 225494, d), dbtestdata.TxidB2T1 + "03", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr5, 225494, d), dbtestdata.TxidB2T3 + "00" + dbtestdata.TxidB2T3 + "01", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.AddrA, 225494, d), dbtestdata.TxidB2T4 + "00", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr4, 225494, d), dbtestdata.TxidB2T2 + "03", nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr1, 225493, d), txIndexesHex(dbtestdata.TxidB1T1, []int32{0}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr2, 225493, d), txIndexesHex(dbtestdata.TxidB1T1, []int32{1}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr3, 225493, d), txIndexesHex(dbtestdata.TxidB1T2, []int32{0}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr4, 225493, d), txIndexesHex(dbtestdata.TxidB1T2, []int32{1}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr5, 225493, d), txIndexesHex(dbtestdata.TxidB1T2, []int32{2}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr6, 225494, d), txIndexesHex(dbtestdata.TxidB2T1, []int32{0}) + txIndexesHex(dbtestdata.TxidB2T2, []int32{^0}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr7, 225494, d), txIndexesHex(dbtestdata.TxidB2T1, []int32{1}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr8, 225494, d), txIndexesHex(dbtestdata.TxidB2T2, []int32{0}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr9, 225494, d), txIndexesHex(dbtestdata.TxidB2T2, []int32{1}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr3, 225494, d), txIndexesHex(dbtestdata.TxidB2T1, []int32{^0}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr2, 225494, d), txIndexesHex(dbtestdata.TxidB2T1, []int32{^1}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr5, 225494, d), txIndexesHex(dbtestdata.TxidB2T3, []int32{0, ^0}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.AddrA, 225494, d), txIndexesHex(dbtestdata.TxidB2T4, []int32{0}), nil},
|
||||
keyPair{addressKeyHex(dbtestdata.Addr4, 225494, d), txIndexesHex(dbtestdata.TxidB2T2, []int32{^1}), nil},
|
||||
}); err != nil {
|
||||
{
|
||||
t.Fatal(err)
|
||||
|
|
@ -373,16 +386,17 @@ func verifyAfterBitcoinTypeBlock2(t *testing.T, d *RocksDB) {
|
|||
}
|
||||
}
|
||||
|
||||
type txidVoutOutput struct {
|
||||
txid string
|
||||
vout int32
|
||||
isOutput bool
|
||||
type txidIndex struct {
|
||||
txid string
|
||||
index int32
|
||||
}
|
||||
|
||||
func verifyGetTransactions(t *testing.T, d *RocksDB, addr string, low, high uint32, wantTxids []txidVoutOutput, wantErr error) {
|
||||
gotTxids := make([]txidVoutOutput, 0)
|
||||
addToTxids := func(txid string, vout int32, isOutput bool) error {
|
||||
gotTxids = append(gotTxids, txidVoutOutput{txid, vout, isOutput})
|
||||
func verifyGetTransactions(t *testing.T, d *RocksDB, addr string, low, high uint32, wantTxids []txidIndex, wantErr error) {
|
||||
gotTxids := make([]txidIndex, 0)
|
||||
addToTxids := func(txid string, height uint32, indexes []int32) error {
|
||||
for _, index := range indexes {
|
||||
gotTxids = append(gotTxids, txidIndex{txid, index})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := d.GetTransactions(addr, low, high, addToTxids); err != nil {
|
||||
|
|
@ -456,21 +470,21 @@ func TestRocksDB_Index_BitcoinType(t *testing.T) {
|
|||
verifyAfterBitcoinTypeBlock2(t, d)
|
||||
|
||||
// get transactions for various addresses / low-high ranges
|
||||
verifyGetTransactions(t, d, dbtestdata.Addr2, 0, 1000000, []txidVoutOutput{
|
||||
txidVoutOutput{dbtestdata.TxidB2T1, 1, false},
|
||||
txidVoutOutput{dbtestdata.TxidB1T1, 1, true},
|
||||
verifyGetTransactions(t, d, dbtestdata.Addr2, 0, 1000000, []txidIndex{
|
||||
{dbtestdata.TxidB2T1, ^1},
|
||||
{dbtestdata.TxidB1T1, 1},
|
||||
}, nil)
|
||||
verifyGetTransactions(t, d, dbtestdata.Addr2, 225493, 225493, []txidVoutOutput{
|
||||
txidVoutOutput{dbtestdata.TxidB1T1, 1, true},
|
||||
verifyGetTransactions(t, d, dbtestdata.Addr2, 225493, 225493, []txidIndex{
|
||||
{dbtestdata.TxidB1T1, 1},
|
||||
}, nil)
|
||||
verifyGetTransactions(t, d, dbtestdata.Addr2, 225494, 1000000, []txidVoutOutput{
|
||||
txidVoutOutput{dbtestdata.TxidB2T1, 1, false},
|
||||
verifyGetTransactions(t, d, dbtestdata.Addr2, 225494, 1000000, []txidIndex{
|
||||
{dbtestdata.TxidB2T1, ^1},
|
||||
}, nil)
|
||||
verifyGetTransactions(t, d, dbtestdata.Addr2, 500000, 1000000, []txidVoutOutput{}, nil)
|
||||
verifyGetTransactions(t, d, dbtestdata.Addr8, 0, 1000000, []txidVoutOutput{
|
||||
txidVoutOutput{dbtestdata.TxidB2T2, 0, true},
|
||||
verifyGetTransactions(t, d, dbtestdata.Addr2, 500000, 1000000, []txidIndex{}, nil)
|
||||
verifyGetTransactions(t, d, dbtestdata.Addr8, 0, 1000000, []txidIndex{
|
||||
{dbtestdata.TxidB2T2, 0},
|
||||
}, nil)
|
||||
verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eBad", 500000, 1000000, []txidVoutOutput{}, errors.New("checksum mismatch"))
|
||||
verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eBad", 500000, 1000000, []txidIndex{}, errors.New("checksum mismatch"))
|
||||
|
||||
// GetBestBlock
|
||||
height, hash, err := d.GetBestBlock()
|
||||
|
|
|
|||
|
|
@ -216,7 +216,7 @@ func (s *SocketIoServer) getAddressTxids(addr []string, opts *addrOpts) (res res
|
|||
lower, higher := uint32(opts.End), uint32(opts.Start)
|
||||
for _, address := range addr {
|
||||
if !opts.QueryMempoolOnly {
|
||||
err = s.db.GetTransactions(address, lower, higher, func(txid string, vout int32, isOutput bool) error {
|
||||
err = s.db.GetTransactions(address, lower, higher, func(txid string, height uint32, indexes []int32) error {
|
||||
txids = append(txids, txid)
|
||||
return nil
|
||||
})
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@
|
|||
<td class="data">{{formatAmount $addr.BalanceSat}} {{$cs}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Non-contract Transactions</td>
|
||||
<td>Transactions</td>
|
||||
<td class="data">{{$addr.TxApperances}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
|
|
|
|||
Loading…
Reference in New Issue