spacecruft
/
blockbook
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implement index v3 for ethereum type coin - WIP

ethereum
Martin Boehm 2018-11-23 14:08:10 +01:00
parent fad7ea326c
commit eb524c2226
11 changed files with 706 additions and 204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
bchain/baseparser.go
View File

@ -50,18 +50,18 @@ func (p *BaseParser) AmountToBigInt(n json.Number) (big.Int, error) {
return r, nil
}
// AmountToDecimalString converts amount in big.Int to string with decimal point in the correct place
func (p *BaseParser) AmountToDecimalString(a *big.Int) string {
// AmountToDecimalString converts amount in big.Int to string with decimal point in the place defined by the parameter d
func AmountToDecimalString(a *big.Int, d int) string {
n := a.String()
var s string
if n[0] == '-' {
n = n[1:]
s = "-"
}
if len(n) <= p.AmountDecimalPoint {
n = zeros[:p.AmountDecimalPoint-len(n)+1] + n
if len(n) <= d {
n = zeros[:d-len(n)+1] + n
}
i := len(n) - p.AmountDecimalPoint
i := len(n) - d
ad := strings.TrimRight(n[i:], "0")
if len(ad) > 0 {
n = n[:i] + "." + ad
@ -71,6 +71,11 @@ func (p *BaseParser) AmountToDecimalString(a *big.Int) string {
return s + n
}
// AmountToDecimalString converts amount in big.Int to string with decimal point in the correct place
func (p *BaseParser) AmountToDecimalString(a *big.Int) string {
return AmountToDecimalString(a, p.AmountDecimalPoint)
}
// ParseTxFromJson parses JSON message containing transaction and returns Tx struct
func (p *BaseParser) ParseTxFromJson(msg json.RawMessage) (*Tx, error) {
var tx Tx

25
bchain/coins/eth/erc20.go
View File

@ -25,25 +25,26 @@ var erc20abi = `[{"constant":true,"inputs":[],"name":"name","outputs":[{"name":"
// doing the parsing/processing without using go-ethereum/accounts/abi library, it is simple to get data from Transfer event
const erc20EventTransferSignature = "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"
type erc20Transfer struct {
Contract ethcommon.Address
From ethcommon.Address
To ethcommon.Address
// Erc20Transfer contains a single Erc20 token transfer
type Erc20Transfer struct {
Contract string
From string
To string
Tokens big.Int
}
func addressFromPaddedHex(s string) (*ethcommon.Address, error) {
func addressFromPaddedHex(s string) (string, error) {
var t big.Int
_, ok := t.SetString(s, 0)
if !ok {
return nil, errors.New("Data is not a number")
return "", errors.New("Data is not a number")
}
a := ethcommon.BigToAddress(&t)
return &a, nil
return a.String(), nil
}
func erc20GetTransfersFromLog(logs []*rpcLog) ([]erc20Transfer, error) {
var r []erc20Transfer
func erc20GetTransfersFromLog(logs []*rpcLog) ([]Erc20Transfer, error) {
var r []Erc20Transfer
for _, l := range logs {
if len(l.Topics) == 3 && l.Topics[0] == erc20EventTransferSignature {
var t big.Int
@ -59,10 +60,10 @@ func erc20GetTransfersFromLog(logs []*rpcLog) ([]erc20Transfer, error) {
if err != nil {
return nil, err
}
r = append(r, erc20Transfer{
r = append(r, Erc20Transfer{
Contract: l.Address,
From: *from,
To: *to,
From: from,
To: to,
Tokens: t,
})
}

42
bchain/coins/eth/erc20_test.go
View File

@ -3,25 +3,24 @@
package eth
import (
fmt "fmt"
"math/big"
"reflect"
"strings"
"testing"
ethcommon "github.com/ethereum/go-ethereum/common"
)
func TestErc20_erc20GetTransfersFromLog(t *testing.T) {
tests := []struct {
name string
args []*rpcLog
want []erc20Transfer
want []Erc20Transfer
wantErr bool
}{
{
name: "1",
args: []*rpcLog{
&rpcLog{
Address: ethcommon.HexToAddress("0x76a45e8976499ab9ae223cc584019341d5a84e96"),
Address: "0x76a45e8976499ab9ae223cc584019341d5a84e96",
Topics: []string{
"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef",
"0x0000000000000000000000002aacf811ac1a60081ea39f7783c0d26c500871a8",
@ -30,11 +29,11 @@ func TestErc20_erc20GetTransfersFromLog(t *testing.T) {
Data: "0x0000000000000000000000000000000000000000000000000000000000000123",
},
},
want: []erc20Transfer{
want: []Erc20Transfer{
{
Contract: ethcommon.HexToAddress("0x76a45e8976499ab9ae223cc584019341d5a84e96"),
From: ethcommon.HexToAddress("0x2aacf811ac1a60081ea39f7783c0d26c500871a8"),
To: ethcommon.HexToAddress("0xe9a5216ff992cfa01594d43501a56e12769eb9d2"),
Contract: "0x76a45e8976499ab9ae223cc584019341d5a84e96",
From: "0x2aacf811ac1a60081ea39f7783c0d26c500871a8",
To: "0xe9a5216ff992cfa01594d43501a56e12769eb9d2",
Tokens: *big.NewInt(0x123),
},
},
@ -43,7 +42,7 @@ func TestErc20_erc20GetTransfersFromLog(t *testing.T) {
name: "2",
args: []*rpcLog{
&rpcLog{ // Transfer
Address: ethcommon.HexToAddress("0x0d0f936ee4c93e25944694d6c121de94d9760f11"),
Address: "0x0d0f936ee4c93e25944694d6c121de94d9760f11",
Topics: []string{
"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef",
"0x0000000000000000000000006f44cceb49b4a5812d54b6f494fc2febf25511ed",
@ -52,7 +51,7 @@ func TestErc20_erc20GetTransfersFromLog(t *testing.T) {
Data: "0x0000000000000000000000000000000000000000000000006a8313d60b1f606b",
},
&rpcLog{ // Transfer
Address: ethcommon.HexToAddress("0xc778417e063141139fce010982780140aa0cd5ab"),
Address: "0xc778417e063141139fce010982780140aa0cd5ab",
Topics: []string{
"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef",
"0x0000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d",
@ -61,7 +60,7 @@ func TestErc20_erc20GetTransfersFromLog(t *testing.T) {
Data: "0x000000000000000000000000000000000000000000000000000308fd0e798ac0",
},
&rpcLog{ // not Transfer
Address: ethcommon.HexToAddress("0x479cc461fecd078f766ecc58533d6f69580cf3ac"),
Address: "0x479cc461fecd078f766ecc58533d6f69580cf3ac",
Topics: []string{
"0x0d0b9391970d9a25552f37d436d2aae2925e2bfe1b2a923754bada030c498cb3",
"0x0000000000000000000000006f44cceb49b4a5812d54b6f494fc2febf25511ed",
@ -71,7 +70,7 @@ func TestErc20_erc20GetTransfersFromLog(t *testing.T) {
Data: "0x0000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d0000000000000",
},
&rpcLog{ // not Transfer
Address: ethcommon.HexToAddress("0x0d0f936ee4c93e25944694d6c121de94d9760f11"),
Address: "0x0d0f936ee4c93e25944694d6c121de94d9760f11",
Topics: []string{
"0x0d0b9391970d9a25552f37d436d2aae2925e2bfe1b2a923754bada030c498cb3",
"0x0000000000000000000000007b62eb7fe80350dc7ec945c0b73242cb9877fb1b",
@ -80,17 +79,17 @@ func TestErc20_erc20GetTransfersFromLog(t *testing.T) {
Data: "0x0000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d000000000000000000000000c778417e063141139fce010982780140aa0cd5ab0000000000000000000000000d0f936ee4c93e25944694d6c121de94d9760f1100000000000000000000000000000000000000000000000000031855667df7a80000000000000000000000000000000000000000000000006a8313d60b1f800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
},
},
want: []erc20Transfer{
want: []Erc20Transfer{
{
Contract: ethcommon.HexToAddress("0x0d0f936ee4c93e25944694d6c121de94d9760f11"),
From: ethcommon.HexToAddress("0x6f44cceb49b4a5812d54b6f494fc2febf25511ed"),
To: ethcommon.HexToAddress("0x4bda106325c335df99eab7fe363cac8a0ba2a24d"),
Contract: "0x0d0f936ee4c93e25944694d6c121de94d9760f11",
From: "0x6f44cceb49b4a5812d54b6f494fc2febf25511ed",
To: "0x4bda106325c335df99eab7fe363cac8a0ba2a24d",
Tokens: *big.NewInt(0x6a8313d60b1f606b),
},
{
Contract: ethcommon.HexToAddress("0xc778417e063141139fce010982780140aa0cd5ab"),
From: ethcommon.HexToAddress("0x4bda106325c335df99eab7fe363cac8a0ba2a24d"),
To: ethcommon.HexToAddress("0x6f44cceb49b4a5812d54b6f494fc2febf25511ed"),
Contract: "0xc778417e063141139fce010982780140aa0cd5ab",
From: "0x4bda106325c335df99eab7fe363cac8a0ba2a24d",
To: "0x6f44cceb49b4a5812d54b6f494fc2febf25511ed",
Tokens: *big.NewInt(0x308fd0e798ac0),
},
},
@ -103,7 +102,8 @@ func TestErc20_erc20GetTransfersFromLog(t *testing.T) {
t.Errorf("erc20GetTransfersFromLog error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
// the addresses could have different case
if strings.ToLower(fmt.Sprint(got)) != strings.ToLower(fmt.Sprint(tt.want)) {
t.Errorf("erc20GetTransfersFromLog = %+v, want %+v", got, tt.want)
}
})

70
bchain/coins/eth/ethparser.go
View File

@ -7,21 +7,24 @@ import (
"math/big"
"strconv"
ethcommon "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/golang/glog"
"github.com/golang/protobuf/proto"
"github.com/juju/errors"
)
// EthereumTypeAddressDescriptorLen - in case of EthereumType, the AddressDescriptor has fixed length
const EthereumTypeAddressDescriptorLen = 20
// EthereumParser handle
type EthereumParser struct {
*bchain.BaseParser
}
// NewEthereumParser returns new EthereumParser instance
func NewEthereumParser() *EthereumParser {
func NewEthereumParser(b int) *EthereumParser {
return &EthereumParser{&bchain.BaseParser{
BlockAddressesToKeep: 0,
BlockAddressesToKeep: b,
AmountDecimalPoint: 18,
}}
}
@ -54,9 +57,9 @@ type rpcTransaction struct {
}
type rpcLog struct {
Address ethcommon.Address `json:"address"`
Topics []string `json:"topics"`
Data string `json:"data"`
Address string `json:"address"`
Topics []string `json:"topics"`
Data string `json:"data"`
}
type rpcLogWithTxHash struct {
@ -171,7 +174,10 @@ func (p *EthereumParser) GetAddrDescFromAddress(address string) (bchain.AddressD
if has0xPrefix(address) {
address = address[2:]
}
if len(address) == 0 {
if len(address) != EthereumTypeAddressDescriptorLen*2 {
if len(address) != 0 {
glog.Warning("Ignoring address ", address)
}
return nil, bchain.ErrAddressMissing
}
if len(address)&1 == 1 {
@ -278,6 +284,10 @@ func (p *EthereumParser) PackTx(tx *bchain.Tx, height uint32, blockTime int64) (
}
ptLogs := make([]*ProtoCompleteTransaction_ReceiptType_LogType, len(r.Receipt.Logs))
for i, l := range r.Receipt.Logs {
a, err := hexutil.Decode(l.Address)
if err != nil {
return nil, errors.Annotatef(err, "Address cannot be decoded %v", l)
}
d, err := hexutil.Decode(l.Data)
if err != nil {
return nil, errors.Annotatef(err, "Data cannot be decoded %v", l)
@ -290,7 +300,7 @@ func (p *EthereumParser) PackTx(tx *bchain.Tx, height uint32, blockTime int64) (
}
}
ptLogs[i] = &ProtoCompleteTransaction_ReceiptType_LogType{
Address: l.Address.Bytes(),
Address: a,
Data: d,
Topics: t,
}
@ -332,7 +342,7 @@ func (p *EthereumParser) UnpackTx(buf []byte) (*bchain.Tx, uint32, error) {
topics[j] = hexutil.Encode(t)
}
logs[i] = &rpcLog{
Address: ethcommon.BytesToAddress(l.Address),
Address: hexutil.Encode(l.Address),
Data: hexutil.Encode(l.Data),
Topics: topics,
}
@ -388,19 +398,39 @@ func (p *EthereumParser) GetChainType() bchain.ChainType {
// GetHeightFromTx returns ethereum specific data from bchain.Tx
func GetHeightFromTx(tx *bchain.Tx) (uint32, error) {
// TODO - temporary implementation - will use bchain.Tx.SpecificData field
b, err := hex.DecodeString(tx.Hex)
if err != nil {
return 0, err
var bn string
csd, ok := tx.CoinSpecificData.(completeTransaction)
if !ok {
b, err := hex.DecodeString(tx.Hex)
if err != nil {
return 0, err
}
var ct completeTransaction
err = json.Unmarshal(b, &ct)
if err != nil {
return 0, err
}
bn = ct.Tx.BlockNumber
} else {
bn = csd.Tx.BlockNumber
}
var ct completeTransaction
var n uint64
err = json.Unmarshal(b, &ct)
n, err := hexutil.DecodeUint64(bn)
if err != nil {
return 0, err
}
if n, err = hexutil.DecodeUint64(ct.Tx.BlockNumber); err != nil {
return 0, errors.Annotatef(err, "BlockNumber %v", ct.Tx.BlockNumber)
return 0, errors.Annotatef(err, "BlockNumber %v", bn)
}
return uint32(n), nil
}
// GetErc20FromTx returns Erc20 data from bchain.Tx
func GetErc20FromTx(tx *bchain.Tx) ([]Erc20Transfer, error) {
var r []Erc20Transfer
var err error
csd, ok := tx.CoinSpecificData.(completeTransaction)
if ok {
r, err = erc20GetTransfersFromLog(csd.Receipt.Logs)
if err != nil {
return nil, err
}
}
return r, nil
}

13
bchain/coins/eth/ethparser_test.go
View File

@ -32,9 +32,10 @@ func TestEthParser_GetAddrDescFromAddress(t *testing.T) {
want: "47526228d673e9f079630d6cdaff5a2ed13e0e60",
},
{
name: "odd address",
args: args{address: "7526228d673e9f079630d6cdaff5a2ed13e0e60"},
want: "07526228d673e9f079630d6cdaff5a2ed13e0e60",
name: "address of wrong length",
args: args{address: "7526228d673e9f079630d6cdaff5a2ed13e0e60"},
want: "",
wantErr: true,
},
{
name: "ErrAddressMissing",
@ -51,7 +52,7 @@ func TestEthParser_GetAddrDescFromAddress(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
p := NewEthereumParser()
p := NewEthereumParser(1)
got, err := p.GetAddrDescFromAddress(tt.args.address)
if (err != nil) != tt.wantErr {
t.Errorf("EthParser.GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr)
@ -146,7 +147,7 @@ func TestEthereumParser_PackTx(t *testing.T) {
want: testTxPacked2,
},
}
p := NewEthereumParser()
p := NewEthereumParser(1)
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := p.PackTx(tt.args.tx, tt.args.height, tt.args.blockTime)
@ -187,7 +188,7 @@ func TestEthereumParser_UnpackTx(t *testing.T) {
want1: 2871048,
},
}
p := NewEthereumParser()
p := NewEthereumParser(1)
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
b, err := hex.DecodeString(tt.args.hex)

15
bchain/coins/eth/ethrpc.go
View File

@ -31,10 +31,11 @@ const (
// Configuration represents json config file
type Configuration struct {
CoinName string `json:"coin_name"`
CoinShortcut string `json:"coin_shortcut"`
RPCURL string `json:"rpc_url"`
RPCTimeout int `json:"rpc_timeout"`
CoinName string `json:"coin_name"`
CoinShortcut string `json:"coin_shortcut"`
RPCURL string `json:"rpc_url"`
RPCTimeout int `json:"rpc_timeout"`
BlockAddressesToKeep int `json:"block_addresses_to_keep"`
}
// EthereumRPC is an interface to JSON-RPC eth service.
@ -65,6 +66,10 @@ func NewEthereumRPC(config json.RawMessage, pushHandler func(bchain.Notification
if err != nil {
return nil, errors.Annotatef(err, "Invalid configuration file")
}
// keep at least 100 mappings block->addresses to allow rollback
if c.BlockAddressesToKeep < 100 {
c.BlockAddressesToKeep = 100
}
rc, err := rpc.Dial(c.RPCURL)
if err != nil {
return nil, err
@ -78,7 +83,7 @@ func NewEthereumRPC(config json.RawMessage, pushHandler func(bchain.Notification
}
// always create parser
s.Parser = NewEthereumParser()
s.Parser = NewEthereumParser(c.BlockAddressesToKeep)
s.timeout = time.Duration(c.RPCTimeout) * time.Second
// detect ethereum classic

190
db/rocksdb.go
View File

@ -59,13 +59,21 @@ const (
cfDefault = iota
cfHeight
cfAddresses
cfTxAddresses
cfAddressBalance
cfBlockTxs
cfTransactions
// BitcoinType
cfAddressBalance
cfTxAddresses
// EthereumType
cfAddressContracts = cfAddressBalance
)
var cfNames = []string{"default", "height", "addresses", "txAddresses", "addressBalance", "blockTxs", "transactions"}
// common columns
var cfNames = []string{"default", "height", "addresses", "blockTxs", "transactions"}
// type specific columns
var cfNamesBitcoinType = []string{"addressBalance", "txAddresses"}
var cfNamesEthereumType = []string{"addressContracts"}
func openDB(path string, c *gorocksdb.Cache, openFiles int) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) {
// opts with bloom filter
@ -73,9 +81,14 @@ func openDB(path string, c *gorocksdb.Cache, openFiles int) (*gorocksdb.DB, []*g
// opts for addresses without bloom filter
// from documentation: if most of your queries are executed using iterators, you shouldn't set bloom filter
optsAddresses := createAndSetDBOptions(0, c, openFiles)
// default, height, addresses, txAddresses, addressBalance, blockTxids, transactions
fcOptions := []*gorocksdb.Options{opts, opts, optsAddresses, opts, opts, opts, opts}
db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions)
// default, height, addresses, blockTxids, transactions
cfOptions := []*gorocksdb.Options{opts, opts, optsAddresses, opts, opts}
// append type specific options
count := len(cfNames) - len(cfOptions)
for i := 0; i < count; i++ {
cfOptions = append(cfOptions, opts)
}
db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, cfOptions)
if err != nil {
return nil, nil, err
}
@ -86,6 +99,16 @@ func openDB(path string, c *gorocksdb.Cache, openFiles int) (*gorocksdb.DB, []*g
// needs to be called to release it.
func NewRocksDB(path string, cacheSize, maxOpenFiles int, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) {
glog.Infof("rocksdb: opening %s, required data version %v, cache size %v, max open files %v", path, dbVersion, cacheSize, maxOpenFiles)
chainType := parser.GetChainType()
if chainType == bchain.ChainBitcoinType {
cfNames = append(cfNames, cfNamesBitcoinType...)
} else if chainType == bchain.ChainEthereumType {
cfNames = append(cfNames, cfNamesEthereumType...)
} else {
return nil, errors.New("Unknown chain type")
}
c := gorocksdb.NewLRUCache(cacheSize)
db, cfh, err := openDB(path, c, maxOpenFiles)
if err != nil {
@ -276,21 +299,18 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error {
if err := d.writeHeightFromBlock(wb, block, op); err != nil {
return err
}
addresses := make(map[string][]outpoint)
if chainType == bchain.ChainBitcoinType {
if op == opDelete {
// block does not contain mapping tx-> input address, which is necessary to recreate
// unspentTxs; therefore it is not possible to DisconnectBlocks this way
return errors.New("DisconnectBlock is not supported for BitcoinType chains")
}
addresses := make(map[string][]outpoint)
txAddressesMap := make(map[string]*TxAddresses)
balances := make(map[string]*AddrBalance)
if err := d.processAddressesBitcoinType(block, addresses, txAddressesMap, balances); err != nil {
return err
}
if err := d.storeAddresses(wb, block.Height, addresses); err != nil {
return err
}
if err := d.storeTxAddresses(wb, txAddressesMap); err != nil {
return err
}
@ -301,12 +321,23 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error {
return err
}
} else if chainType == bchain.ChainEthereumType {
if err := d.writeAddressesEthereumType(wb, block, op); err != nil {
addressContracts := make(map[string]*AddrContracts)
blockTxs, err := d.processAddressesEthereumType(block, addresses, addressContracts)
if err != nil {
return err
}
if err := d.storeAddressContracts(wb, addressContracts); err != nil {
return err
}
if err := d.storeAndCleanupBlockTxsEthereumType(wb, block, blockTxs); err != nil {
return err
}
} else {
return errors.New("Unknown chain type")
}
if err := d.storeAddresses(wb, block.Height, addresses); err != nil {
return err
}
return d.db.Write(d.wo, wb)
}
@ -579,6 +610,26 @@ func (d *RocksDB) storeBalances(wb *gorocksdb.WriteBatch, abm map[string]*AddrBa
return nil
}
func (d *RocksDB) cleanupBlockTxs(wb *gorocksdb.WriteBatch, block *bchain.Block) error {
keep := d.chainParser.KeepBlockAddresses()
// cleanup old block address
if block.Height > uint32(keep) {
for rh := block.Height - uint32(keep); rh < block.Height; rh-- {
key := packUint(rh)
val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxs], key)
if err != nil {
return err
}
if val.Size() == 0 {
break
}
val.Free()
d.db.DeleteCF(d.wo, d.cfh[cfBlockTxs], key)
}
}
return nil
}
func (d *RocksDB) storeAndCleanupBlockTxs(wb *gorocksdb.WriteBatch, block *bchain.Block) error {
pl := d.chainParser.PackedTxidLen()
buf := make([]byte, 0, pl*len(block.Txs))
@ -612,23 +663,7 @@ func (d *RocksDB) storeAndCleanupBlockTxs(wb *gorocksdb.WriteBatch, block *bchai
}
key := packUint(block.Height)
wb.PutCF(d.cfh[cfBlockTxs], key, buf)
keep := d.chainParser.KeepBlockAddresses()
// cleanup old block address
if block.Height > uint32(keep) {
for rh := block.Height - uint32(keep); rh < block.Height; rh-- {
key = packUint(rh)
val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxs], key)
if err != nil {
return err
}
if val.Size() == 0 {
break
}
val.Free()
d.db.DeleteCF(d.wo, d.cfh[cfBlockTxs], key)
}
}
return nil
return d.cleanupBlockTxs(wb, block)
}
func (d *RocksDB) getBlockTxs(height uint32) ([]blockTxs, error) {
@ -844,74 +879,6 @@ func (d *RocksDB) unpackNOutpoints(buf []byte) ([]outpoint, int, error) {
return outpoints, p, nil
}
func (d *RocksDB) addAddrDescToRecords(op int, wb *gorocksdb.WriteBatch, records map[string][]outpoint, addrDesc bchain.AddressDescriptor, btxid []byte, vout int32, bh uint32) error {
if len(addrDesc) > 0 {
if len(addrDesc) > maxAddrDescLen {
glog.Infof("rocksdb: block %d, skipping addrDesc of length %d", bh, len(addrDesc))
} else {
strAddrDesc := string(addrDesc)
records[strAddrDesc] = append(records[strAddrDesc], outpoint{
btxID: btxid,
index: vout,
})
if op == opDelete {
// remove transactions from cache
d.internalDeleteTx(wb, btxid)
}
}
}
return nil
}
func (d *RocksDB) writeAddressesEthereumType(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error {
addresses := make(map[string][]outpoint)
for _, tx := range block.Txs {
btxID, err := d.chainParser.PackTxid(tx.Txid)
if err != nil {
return err
}
for _, output := range tx.Vout {
addrDesc, err := d.chainParser.GetAddrDescFromVout(&output)
if err != nil {
// do not log ErrAddressMissing, transactions can be without to address (for example eth contracts)
if err != bchain.ErrAddressMissing {
glog.Warningf("rocksdb: addrDesc: %v - height %d, tx %v, output %v", err, block.Height, tx.Txid, output)
}
continue
}
err = d.addAddrDescToRecords(op, wb, addresses, addrDesc, btxID, int32(output.N), block.Height)
if err != nil {
return err
}
}
// store inputs in format txid ^index
for _, input := range tx.Vin {
for i, a := range input.Addresses {
addrDesc, err := d.chainParser.GetAddrDescFromAddress(a)
if err != nil {
glog.Warningf("rocksdb: addrDesc: %v - %d %s", err, block.Height, addrDesc)
continue
}
err = d.addAddrDescToRecords(op, wb, addresses, addrDesc, btxID, int32(^i), block.Height)
if err != nil {
return err
}
}
}
}
for addrDesc, outpoints := range addresses {
key := packAddressKey(bchain.AddressDescriptor(addrDesc), block.Height)
switch op {
case opInsert:
val := d.packOutpoints(outpoints)
wb.PutCF(d.cfh[cfAddresses], key, val)
case opDelete:
wb.DeleteCF(d.cfh[cfAddresses], key)
}
}
return nil
}
// Block index
// BlockInfo holds information about blocks kept in column height
@ -1229,37 +1196,6 @@ func (d *RocksDB) DisconnectBlockRangeBitcoinType(lower uint32, higher uint32) e
return err
}
// DisconnectBlockRangeNonUTXO performs full range scan to remove a range of blocks
// it is very slow operation
func (d *RocksDB) DisconnectBlockRangeNonUTXO(lower uint32, higher uint32) error {
glog.Infof("db: disconnecting blocks %d-%d", lower, higher)
addrKeys, _, err := d.allAddressesScan(lower, higher)
if err != nil {
return err
}
glog.Infof("rocksdb: about to disconnect %d addresses ", len(addrKeys))
wb := gorocksdb.NewWriteBatch()
defer wb.Destroy()
for _, addrKey := range addrKeys {
if glog.V(2) {
glog.Info("address ", hex.EncodeToString(addrKey))
}
// delete address:height from the index
wb.DeleteCF(d.cfh[cfAddresses], addrKey)
}
for height := lower; height <= higher; height++ {
if glog.V(2) {
glog.Info("height ", height)
}
wb.DeleteCF(d.cfh[cfHeight], packUint(height))
}
err = d.db.Write(d.wo, wb)
if err == nil {
glog.Infof("rocksdb: blocks %d-%d disconnected", lower, higher)
}
return err
}
func dirSize(path string) (int64, error) {
var size int64
err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {

272
db/rocksdb_ethereumtype.go 100644
View File

@ -0,0 +1,272 @@
package db
import (
"blockbook/bchain"
"blockbook/bchain/coins/eth"
"bytes"
"encoding/hex"
"github.com/bsm/go-vlq"
"github.com/golang/glog"
"github.com/juju/errors"
"github.com/tecbot/gorocksdb"
)
// AddrContract is Contract address with number of transactions done by given address
type AddrContract struct {
Contract bchain.AddressDescriptor
Txs uint
}
// AddrContracts is array of contracts with
type AddrContracts struct {
EthTxs uint
Contracts []AddrContract
}
func (d *RocksDB) storeAddressContracts(wb *gorocksdb.WriteBatch, acm map[string]*AddrContracts) error {
buf := make([]byte, 64)
varBuf := make([]byte, vlq.MaxLen64)
for addrDesc, acs := range acm {
// address with 0 contracts is removed from db - happens on disconnect
if acs == nil || (acs.EthTxs == 0 && len(acs.Contracts) == 0) {
wb.DeleteCF(d.cfh[cfAddressContracts], bchain.AddressDescriptor(addrDesc))
} else {
buf = buf[:0]
l := packVaruint(acs.EthTxs, varBuf)
buf = append(buf, varBuf[:l]...)
for _, ac := range acs.Contracts {
buf = append(buf, ac.Contract...)
l = packVaruint(ac.Txs, varBuf)
buf = append(buf, varBuf[:l]...)
}
wb.PutCF(d.cfh[cfAddressContracts], bchain.AddressDescriptor(addrDesc), buf)
}
}
return nil
}
// GetAddrDescContracts returns AddrContracts for given addrDesc
func (d *RocksDB) GetAddrDescContracts(addrDesc bchain.AddressDescriptor) (*AddrContracts, error) {
val, err := d.db.GetCF(d.ro, d.cfh[cfAddressContracts], addrDesc)
if err != nil {
return nil, err
}
defer val.Free()
buf := val.Data()
if len(buf) == 0 {
return nil, nil
}
c := make([]AddrContract, 0, 4)
et, l := unpackVaruint(buf)
buf = buf[l:]
for len(buf) > 0 {
if len(buf) < eth.EthereumTypeAddressDescriptorLen {
return nil, errors.New("Invalid data stored in cfAddressContracts for AddrDesc " + addrDesc.String())
}
txs, l := unpackVaruint(buf[eth.EthereumTypeAddressDescriptorLen:])
contract := make(bchain.AddressDescriptor, eth.EthereumTypeAddressDescriptorLen)
copy(contract, buf[:eth.EthereumTypeAddressDescriptorLen])
c = append(c, AddrContract{
Contract: contract,
Txs: txs,
})
buf = buf[eth.EthereumTypeAddressDescriptorLen+l:]
}
return &AddrContracts{
EthTxs: et,
Contracts: c}, nil
}
func (d *RocksDB) addToAddressesAndContractsEthereumType(addrDesc bchain.AddressDescriptor, btxID []byte, index int32, contract bchain.AddressDescriptor, addresses map[string][]outpoint, addressContracts map[string]*AddrContracts) error {
var err error
strAddrDesc := string(addrDesc)
ac, e := addressContracts[strAddrDesc]
if !e {
ac, err = d.GetAddrDescContracts(addrDesc)
if err != nil {
return err
}
if ac == nil {
ac = &AddrContracts{}
}
addressContracts[strAddrDesc] = ac
d.cbs.balancesMiss++
} else {
d.cbs.balancesHit++
}
if contract == nil {
ac.EthTxs++
} else {
// locate the contract and set i to the index in the array of contracts
var i int
var found bool
for i = range ac.Contracts {
if bytes.Equal(contract, ac.Contracts[i].Contract) {
found = true
break
}
}
if !found {
i = len(ac.Contracts)
ac.Contracts = append(ac.Contracts, AddrContract{Contract: contract})
}
// index 0 is for ETH transfers, contract indexes start with 1
if index < 0 {
index = ^int32(i + 1)
} else {
index = int32(i + 1)
}
ac.Contracts[i].Txs++
}
addresses[strAddrDesc] = append(addresses[strAddrDesc], outpoint{
btxID: btxID,
index: index,
})
return nil
}
type ethBlockTxContract struct {
addr, contract bchain.AddressDescriptor
}
type ethBlockTx struct {
btxID []byte
from, to bchain.AddressDescriptor
contracts []ethBlockTxContract
}
func (d *RocksDB) processAddressesEthereumType(block *bchain.Block, addresses map[string][]outpoint, addressContracts map[string]*AddrContracts) ([]ethBlockTx, error) {
blockTxs := make([]ethBlockTx, len(block.Txs))
for txi, tx := range block.Txs {
btxID, err := d.chainParser.PackTxid(tx.Txid)
if err != nil {
return nil, err
}
blockTx := &blockTxs[txi]
blockTx.btxID = btxID
// there is only one output address in EthereumType transaction, store it in format txid 0
if len(tx.Vout) == 1 && len(tx.Vout[0].ScriptPubKey.Addresses) == 1 {
addrDesc, err := d.chainParser.GetAddrDescFromAddress(tx.Vout[0].ScriptPubKey.Addresses[0])
if err != nil {
// do not log ErrAddressMissing, transactions can be without to address (for example eth contracts)
if err != bchain.ErrAddressMissing {
glog.Warningf("rocksdb: addrDesc: %v - height %d, tx %v, output", err, block.Height, tx.Txid)
}
continue
}
if err = d.addToAddressesAndContractsEthereumType(addrDesc, btxID, 0, nil, addresses, addressContracts); err != nil {
return nil, err
}
blockTx.to = addrDesc
}
// there is only one input address in EthereumType transaction, store it in format txid ^0
if len(tx.Vin) == 1 && len(tx.Vin[0].Addresses) == 1 {
addrDesc, err := d.chainParser.GetAddrDescFromAddress(tx.Vin[0].Addresses[0])
if err != nil {
if err != bchain.ErrAddressMissing {
glog.Warningf("rocksdb: addrDesc: %v - height %d, tx %v, input", err, block.Height, tx.Txid)
}
continue
}
if err = d.addToAddressesAndContractsEthereumType(addrDesc, btxID, ^int32(0), nil, addresses, addressContracts); err != nil {
return nil, err
}
blockTx.from = addrDesc
}
// store erc20 transfers
erc20, err := eth.GetErc20FromTx(&tx)
if err != nil {
glog.Warningf("rocksdb: GetErc20FromTx %v - height %d, tx %v", err, block.Height, tx.Txid)
}
blockTx.contracts = make([]ethBlockTxContract, len(erc20)*2)
for i, t := range erc20 {
var contract, from, to bchain.AddressDescriptor
contract, err = d.chainParser.GetAddrDescFromAddress(t.Contract)
if err == nil {
from, err = d.chainParser.GetAddrDescFromAddress(t.From)
if err == nil {
to, err = d.chainParser.GetAddrDescFromAddress(t.To)
}
}
if err != nil {
glog.Warningf("rocksdb: GetErc20FromTx %v - height %d, tx %v, transfer %v", err, block.Height, tx.Txid, t)
continue
}
if err = d.addToAddressesAndContractsEthereumType(from, btxID, ^int32(i), contract, addresses, addressContracts); err != nil {
return nil, err
}
bc := &blockTx.contracts[i*2]
bc.addr = from
bc.contract = contract
if err = d.addToAddressesAndContractsEthereumType(to, btxID, int32(i), contract, addresses, addressContracts); err != nil {
return nil, err
}
bc = &blockTx.contracts[i*2+1]
bc.addr = to
bc.contract = contract
}
}
return blockTxs, nil
}
func (d *RocksDB) storeAndCleanupBlockTxsEthereumType(wb *gorocksdb.WriteBatch, block *bchain.Block, blockTxs []ethBlockTx) error {
pl := d.chainParser.PackedTxidLen()
buf := make([]byte, 0, (pl+2*eth.EthereumTypeAddressDescriptorLen)*len(blockTxs))
varBuf := make([]byte, vlq.MaxLen64)
zeroAddress := make([]byte, eth.EthereumTypeAddressDescriptorLen)
appendAddress := func(a bchain.AddressDescriptor) {
if len(a) != eth.EthereumTypeAddressDescriptorLen {
buf = append(buf, zeroAddress...)
} else {
buf = append(buf, a...)
}
}
for i := range blockTxs {
blockTx := &blockTxs[i]
buf = append(buf, blockTx.btxID...)
appendAddress(blockTx.from)
appendAddress(blockTx.to)
l := packVaruint(uint(len(blockTx.contracts)), varBuf)
buf = append(buf, varBuf[:l]...)
for j := range blockTx.contracts {
c := &blockTx.contracts[j]
appendAddress(c.addr)
appendAddress(c.contract)
}
}
key := packUint(block.Height)
wb.PutCF(d.cfh[cfBlockTxs], key, buf)
return d.cleanupBlockTxs(wb, block)
}
// DisconnectBlockRangeNonUTXO performs full range scan to remove a range of blocks
// it is very slow operation
func (d *RocksDB) DisconnectBlockRangeNonUTXO(lower uint32, higher uint32) error {
glog.Infof("db: disconnecting blocks %d-%d", lower, higher)
addrKeys, _, err := d.allAddressesScan(lower, higher)
if err != nil {
return err
}
glog.Infof("rocksdb: about to disconnect %d addresses ", len(addrKeys))
wb := gorocksdb.NewWriteBatch()
defer wb.Destroy()
for _, addrKey := range addrKeys {
if glog.V(2) {
glog.Info("address ", hex.EncodeToString(addrKey))
}
// delete address:height from the index
wb.DeleteCF(d.cfh[cfAddresses], addrKey)
}
for height := lower; height <= higher; height++ {
if glog.V(2) {
glog.Info("height ", height)
}
wb.DeleteCF(d.cfh[cfHeight], packUint(height))
}
err = d.db.Write(d.wo, wb)
if err == nil {
glog.Infof("rocksdb: blocks %d-%d disconnected", lower, higher)
}
return err
}

183
db/rocksdb_ethereumtype_test.go 100644
View File

@ -0,0 +1,183 @@
// build unittest
package db
import (
"blockbook/bchain/coins/eth"
"blockbook/tests/dbtestdata"
"testing"
)
type testEthereumParser struct {
*eth.EthereumParser
}
func ethereumTestnetParser() *eth.EthereumParser {
return eth.NewEthereumParser(1)
}
func verifyAfterEthereumTypeBlock1(t *testing.T, d *RocksDB, afterDisconnect bool) {
if err := checkColumn(d, cfHeight, []keyPair{
keyPair{
"0041eee8",
"c7b98df95acfd11c51ba25611a39e004fe56c8fdfc1582af99354fcd09c17b11" + uintToHex(1534858022) + varuintToHex(2) + varuintToHex(31839),
nil,
},
}); err != nil {
{
t.Fatal(err)
}
}
// the vout is encoded as signed varint, i.e. value * 2 for positive values, abs(value)*2 + 1 for negative values
if err := checkColumn(d, cfAddresses, []keyPair{
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "01", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "00" + dbtestdata.EthTxidB1T2 + "02", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "01" + dbtestdata.EthTxidB1T2 + "03", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "00", nil},
}); err != nil {
{
t.Fatal(err)
}
}
if err := checkColumn(d, cfAddressContracts, []keyPair{
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser), "01", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser), "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser), "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser), "01", nil},
}); err != nil {
{
t.Fatal(err)
}
}
var blockTxsKp []keyPair
if afterDisconnect {
blockTxsKp = []keyPair{}
} else {
blockTxsKp = []keyPair{
keyPair{
"0041eee8",
dbtestdata.EthTxidB1T1 +
dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "00" +
dbtestdata.EthTxidB1T2 +
dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) +
"02" +
dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) +
dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser),
nil,
},
}
}
if err := checkColumn(d, cfBlockTxs, blockTxsKp); err != nil {
{
t.Fatal(err)
}
}
}
func verifyAfterEthereumTypeBlock2(t *testing.T, d *RocksDB) {
if err := checkColumn(d, cfHeight, []keyPair{
keyPair{
"0041eee8",
"c7b98df95acfd11c51ba25611a39e004fe56c8fdfc1582af99354fcd09c17b11" + uintToHex(1534858022) + varuintToHex(2) + varuintToHex(31839),
nil,
},
keyPair{
"0041eee9",
"00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6" + uintToHex(1534859988) + varuintToHex(2) + varuintToHex(2345678),
nil,
},
}); err != nil {
{
t.Fatal(err)
}
}
// the vout is encoded as signed varint, i.e. value * 2 for positive values, abs(value)*2 + 1 for negative values
if err := checkColumn(d, cfAddresses, []keyPair{
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "01", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "00" + dbtestdata.EthTxidB1T2 + "02", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "01" + dbtestdata.EthTxidB1T2 + "03", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "00", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T1 + "01" + dbtestdata.EthTxidB2T2 + "05" + dbtestdata.EthTxidB2T2 + "02", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr9f, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T1 + "00", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T2 + "01" + dbtestdata.EthTxidB2T2 + "02" + dbtestdata.EthTxidB2T2 + "05" + dbtestdata.EthTxidB2T2 + "04" + dbtestdata.EthTxidB2T2 + "03", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr7b, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T2 + "03" + dbtestdata.EthTxidB2T2 + "04", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract47, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T2 + "00", nil},
}); err != nil {
{
t.Fatal(err)
}
}
if err := checkColumn(d, cfAddressContracts, []keyPair{
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser), "01", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser), "02" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "02" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + "01", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser), "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser), "01", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr9f, d.chainParser), "01", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser), "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + "02" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "02", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr7b, d.chainParser), "00" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + "01", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract47, d.chainParser), "01", nil},
}); err != nil {
{
t.Fatal(err)
}
}
if err := checkColumn(d, cfBlockTxs, []keyPair{
keyPair{
"0041eee9",
dbtestdata.EthTxidB2T1 +
dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr9f, d.chainParser) + "00" +
dbtestdata.EthTxidB2T2 +
dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract47, d.chainParser) +
"08" +
dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) +
dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) +
dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) +
dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) +
dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr7b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) +
dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) +
dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) +
dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr7b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser),
nil,
},
}); err != nil {
{
t.Fatal(err)
}
}
}
// TestRocksDB_Index_EthereumType is an integration test probing the whole indexing functionality for EthereumType chains
// It does the following:
// 1) Connect two blocks (inputs from 2nd block are spending some outputs from the 1st block)
// 2) GetTransactions for various addresses / low-high ranges
// 3) GetBestBlock, GetBlockHash
// 4) Test tx caching functionality
// 5) Disconnect block 2 - expect error
// 6) Disconnect the block 2 using BlockTxs column
// 7) Reconnect block 2 and check
// After each step, the content of DB is examined and any difference against expected state is regarded as failure
func TestRocksDB_Index_EthereumType(t *testing.T) {
d := setupRocksDB(t, &testEthereumParser{
EthereumParser: ethereumTestnetParser(),
})
defer closeAndDestroyRocksDB(t, d)
// connect 1st block - will log warnings about missing UTXO transactions in txAddresses column
block1 := dbtestdata.GetTestEthereumTypeBlock1(d.chainParser)
if err := d.ConnectBlock(block1); err != nil {
t.Fatal(err)
}
verifyAfterEthereumTypeBlock1(t, d, false)
// connect 2nd block - use some outputs from the 1st block as the inputs and 1 input uses tx from the same block
block2 := dbtestdata.GetTestEthereumTypeBlock2(d.chainParser)
if err := d.ConnectBlock(block2); err != nil {
t.Fatal(err)
}
verifyAfterEthereumTypeBlock2(t, d)
}

16
db/rocksdb_test.go
View File

@ -33,6 +33,10 @@ func TestMain(m *testing.M) {
os.Exit(c)
}
type testBitcoinParser struct {
*btc.BitcoinParser
}
func bitcoinTestnetParser() *btc.BitcoinParser {
return btc.NewBitcoinParser(
btc.GetChainParams("test"),
@ -48,7 +52,7 @@ func setupRocksDB(t *testing.T, p bchain.BlockChainParser) *RocksDB {
if err != nil {
t.Fatal(err)
}
is, err := d.LoadInternalState("btc-testnet")
is, err := d.LoadInternalState("coin-unittest")
if err != nil {
t.Fatal(err)
}
@ -151,7 +155,7 @@ func checkColumn(d *RocksDB, col int, kp []keyPair) error {
return nil
}
func verifyAfterBitcoinTypeOBlock1(t *testing.T, d *RocksDB, afterDisconnect bool) {
func verifyAfterBitcoinTypeBlock1(t *testing.T, d *RocksDB, afterDisconnect bool) {
if err := checkColumn(d, cfHeight, []keyPair{
keyPair{
"000370d5",
@ -388,10 +392,6 @@ func verifyGetTransactions(t *testing.T, d *RocksDB, addr string, low, high uint
}
}
type testBitcoinParser struct {
*btc.BitcoinParser
}
// override PackTx and UnpackTx to default BaseParser functionality
// BitcoinParser uses tx hex which is not available for the test transactions
func (p *testBitcoinParser) PackTx(tx *bchain.Tx, height uint32, blockTime int64) ([]byte, error) {
@ -444,7 +444,7 @@ func TestRocksDB_Index_BitcoinType(t *testing.T) {
if err := d.ConnectBlock(block1); err != nil {
t.Fatal(err)
}
verifyAfterBitcoinTypeOBlock1(t, d, false)
verifyAfterBitcoinTypeBlock1(t, d, false)
// connect 2nd block - use some outputs from the 1st block as the inputs and 1 input uses tx from the same block
block2 := dbtestdata.GetTestBitcoinTypeBlock2(d.chainParser)
@ -552,7 +552,7 @@ func TestRocksDB_Index_BitcoinType(t *testing.T) {
if err != nil {
t.Fatal(err)
}
verifyAfterBitcoinTypeOBlock1(t, d, true)
verifyAfterBitcoinTypeBlock1(t, d, true)
if err := checkColumn(d, cfTransactions, []keyPair{}); err != nil {
{
t.Fatal(err)

69
tests/dbtestdata/dbtestdata_ethereumtype.go 100644
View File

File diff suppressed because one or more lines are too long