diff --git a/bchain/baseparser.go b/bchain/baseparser.go index d8a878ae..7d4f3875 100644 --- a/bchain/baseparser.go +++ b/bchain/baseparser.go @@ -50,18 +50,18 @@ func (p *BaseParser) AmountToBigInt(n json.Number) (big.Int, error) { return r, nil } -// AmountToDecimalString converts amount in big.Int to string with decimal point in the correct place -func (p *BaseParser) AmountToDecimalString(a *big.Int) string { +// AmountToDecimalString converts amount in big.Int to string with decimal point in the place defined by the parameter d +func AmountToDecimalString(a *big.Int, d int) string { n := a.String() var s string if n[0] == '-' { n = n[1:] s = "-" } - if len(n) <= p.AmountDecimalPoint { - n = zeros[:p.AmountDecimalPoint-len(n)+1] + n + if len(n) <= d { + n = zeros[:d-len(n)+1] + n } - i := len(n) - p.AmountDecimalPoint + i := len(n) - d ad := strings.TrimRight(n[i:], "0") if len(ad) > 0 { n = n[:i] + "." + ad @@ -71,6 +71,11 @@ func (p *BaseParser) AmountToDecimalString(a *big.Int) string { return s + n } +// AmountToDecimalString converts amount in big.Int to string with decimal point in the correct place +func (p *BaseParser) AmountToDecimalString(a *big.Int) string { + return AmountToDecimalString(a, p.AmountDecimalPoint) +} + // ParseTxFromJson parses JSON message containing transaction and returns Tx struct func (p *BaseParser) ParseTxFromJson(msg json.RawMessage) (*Tx, error) { var tx Tx diff --git a/bchain/coins/eth/erc20.go b/bchain/coins/eth/erc20.go index 0d2a6a5f..1935ba68 100644 --- a/bchain/coins/eth/erc20.go +++ b/bchain/coins/eth/erc20.go @@ -25,25 +25,26 @@ var erc20abi = `[{"constant":true,"inputs":[],"name":"name","outputs":[{"name":" // doing the parsing/processing without using go-ethereum/accounts/abi library, it is simple to get data from Transfer event const erc20EventTransferSignature = "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef" -type erc20Transfer struct { - Contract ethcommon.Address - From ethcommon.Address - To ethcommon.Address +// Erc20Transfer contains a single Erc20 token transfer +type Erc20Transfer struct { + Contract string + From string + To string Tokens big.Int } -func addressFromPaddedHex(s string) (*ethcommon.Address, error) { +func addressFromPaddedHex(s string) (string, error) { var t big.Int _, ok := t.SetString(s, 0) if !ok { - return nil, errors.New("Data is not a number") + return "", errors.New("Data is not a number") } a := ethcommon.BigToAddress(&t) - return &a, nil + return a.String(), nil } -func erc20GetTransfersFromLog(logs []*rpcLog) ([]erc20Transfer, error) { - var r []erc20Transfer +func erc20GetTransfersFromLog(logs []*rpcLog) ([]Erc20Transfer, error) { + var r []Erc20Transfer for _, l := range logs { if len(l.Topics) == 3 && l.Topics[0] == erc20EventTransferSignature { var t big.Int @@ -59,10 +60,10 @@ func erc20GetTransfersFromLog(logs []*rpcLog) ([]erc20Transfer, error) { if err != nil { return nil, err } - r = append(r, erc20Transfer{ + r = append(r, Erc20Transfer{ Contract: l.Address, - From: *from, - To: *to, + From: from, + To: to, Tokens: t, }) } diff --git a/bchain/coins/eth/erc20_test.go b/bchain/coins/eth/erc20_test.go index d806ab9e..1b5b6fbd 100644 --- a/bchain/coins/eth/erc20_test.go +++ b/bchain/coins/eth/erc20_test.go @@ -3,25 +3,24 @@ package eth import ( + fmt "fmt" "math/big" - "reflect" + "strings" "testing" - - ethcommon "github.com/ethereum/go-ethereum/common" ) func TestErc20_erc20GetTransfersFromLog(t *testing.T) { tests := []struct { name string args []*rpcLog - want []erc20Transfer + want []Erc20Transfer wantErr bool }{ { name: "1", args: []*rpcLog{ &rpcLog{ - Address: ethcommon.HexToAddress("0x76a45e8976499ab9ae223cc584019341d5a84e96"), + Address: "0x76a45e8976499ab9ae223cc584019341d5a84e96", Topics: []string{ "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", "0x0000000000000000000000002aacf811ac1a60081ea39f7783c0d26c500871a8", @@ -30,11 +29,11 @@ func TestErc20_erc20GetTransfersFromLog(t *testing.T) { Data: "0x0000000000000000000000000000000000000000000000000000000000000123", }, }, - want: []erc20Transfer{ + want: []Erc20Transfer{ { - Contract: ethcommon.HexToAddress("0x76a45e8976499ab9ae223cc584019341d5a84e96"), - From: ethcommon.HexToAddress("0x2aacf811ac1a60081ea39f7783c0d26c500871a8"), - To: ethcommon.HexToAddress("0xe9a5216ff992cfa01594d43501a56e12769eb9d2"), + Contract: "0x76a45e8976499ab9ae223cc584019341d5a84e96", + From: "0x2aacf811ac1a60081ea39f7783c0d26c500871a8", + To: "0xe9a5216ff992cfa01594d43501a56e12769eb9d2", Tokens: *big.NewInt(0x123), }, }, @@ -43,7 +42,7 @@ func TestErc20_erc20GetTransfersFromLog(t *testing.T) { name: "2", args: []*rpcLog{ &rpcLog{ // Transfer - Address: ethcommon.HexToAddress("0x0d0f936ee4c93e25944694d6c121de94d9760f11"), + Address: "0x0d0f936ee4c93e25944694d6c121de94d9760f11", Topics: []string{ "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", "0x0000000000000000000000006f44cceb49b4a5812d54b6f494fc2febf25511ed", @@ -52,7 +51,7 @@ func TestErc20_erc20GetTransfersFromLog(t *testing.T) { Data: "0x0000000000000000000000000000000000000000000000006a8313d60b1f606b", }, &rpcLog{ // Transfer - Address: ethcommon.HexToAddress("0xc778417e063141139fce010982780140aa0cd5ab"), + Address: "0xc778417e063141139fce010982780140aa0cd5ab", Topics: []string{ "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", "0x0000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d", @@ -61,7 +60,7 @@ func TestErc20_erc20GetTransfersFromLog(t *testing.T) { Data: "0x000000000000000000000000000000000000000000000000000308fd0e798ac0", }, &rpcLog{ // not Transfer - Address: ethcommon.HexToAddress("0x479cc461fecd078f766ecc58533d6f69580cf3ac"), + Address: "0x479cc461fecd078f766ecc58533d6f69580cf3ac", Topics: []string{ "0x0d0b9391970d9a25552f37d436d2aae2925e2bfe1b2a923754bada030c498cb3", "0x0000000000000000000000006f44cceb49b4a5812d54b6f494fc2febf25511ed", @@ -71,7 +70,7 @@ func TestErc20_erc20GetTransfersFromLog(t *testing.T) { Data: "0x0000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d0000000000000", }, &rpcLog{ // not Transfer - Address: ethcommon.HexToAddress("0x0d0f936ee4c93e25944694d6c121de94d9760f11"), + Address: "0x0d0f936ee4c93e25944694d6c121de94d9760f11", Topics: []string{ "0x0d0b9391970d9a25552f37d436d2aae2925e2bfe1b2a923754bada030c498cb3", "0x0000000000000000000000007b62eb7fe80350dc7ec945c0b73242cb9877fb1b", @@ -80,17 +79,17 @@ func TestErc20_erc20GetTransfersFromLog(t *testing.T) { Data: "0x0000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d000000000000000000000000c778417e063141139fce010982780140aa0cd5ab0000000000000000000000000d0f936ee4c93e25944694d6c121de94d9760f1100000000000000000000000000000000000000000000000000031855667df7a80000000000000000000000000000000000000000000000006a8313d60b1f800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", }, }, - want: []erc20Transfer{ + want: []Erc20Transfer{ { - Contract: ethcommon.HexToAddress("0x0d0f936ee4c93e25944694d6c121de94d9760f11"), - From: ethcommon.HexToAddress("0x6f44cceb49b4a5812d54b6f494fc2febf25511ed"), - To: ethcommon.HexToAddress("0x4bda106325c335df99eab7fe363cac8a0ba2a24d"), + Contract: "0x0d0f936ee4c93e25944694d6c121de94d9760f11", + From: "0x6f44cceb49b4a5812d54b6f494fc2febf25511ed", + To: "0x4bda106325c335df99eab7fe363cac8a0ba2a24d", Tokens: *big.NewInt(0x6a8313d60b1f606b), }, { - Contract: ethcommon.HexToAddress("0xc778417e063141139fce010982780140aa0cd5ab"), - From: ethcommon.HexToAddress("0x4bda106325c335df99eab7fe363cac8a0ba2a24d"), - To: ethcommon.HexToAddress("0x6f44cceb49b4a5812d54b6f494fc2febf25511ed"), + Contract: "0xc778417e063141139fce010982780140aa0cd5ab", + From: "0x4bda106325c335df99eab7fe363cac8a0ba2a24d", + To: "0x6f44cceb49b4a5812d54b6f494fc2febf25511ed", Tokens: *big.NewInt(0x308fd0e798ac0), }, }, @@ -103,7 +102,8 @@ func TestErc20_erc20GetTransfersFromLog(t *testing.T) { t.Errorf("erc20GetTransfersFromLog error = %v, wantErr %v", err, tt.wantErr) return } - if !reflect.DeepEqual(got, tt.want) { + // the addresses could have different case + if strings.ToLower(fmt.Sprint(got)) != strings.ToLower(fmt.Sprint(tt.want)) { t.Errorf("erc20GetTransfersFromLog = %+v, want %+v", got, tt.want) } }) diff --git a/bchain/coins/eth/ethparser.go b/bchain/coins/eth/ethparser.go index ff13bd24..7b93fa7d 100644 --- a/bchain/coins/eth/ethparser.go +++ b/bchain/coins/eth/ethparser.go @@ -7,21 +7,24 @@ import ( "math/big" "strconv" - ethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/golang/glog" "github.com/golang/protobuf/proto" "github.com/juju/errors" ) +// EthereumTypeAddressDescriptorLen - in case of EthereumType, the AddressDescriptor has fixed length +const EthereumTypeAddressDescriptorLen = 20 + // EthereumParser handle type EthereumParser struct { *bchain.BaseParser } // NewEthereumParser returns new EthereumParser instance -func NewEthereumParser() *EthereumParser { +func NewEthereumParser(b int) *EthereumParser { return &EthereumParser{&bchain.BaseParser{ - BlockAddressesToKeep: 0, + BlockAddressesToKeep: b, AmountDecimalPoint: 18, }} } @@ -54,9 +57,9 @@ type rpcTransaction struct { } type rpcLog struct { - Address ethcommon.Address `json:"address"` - Topics []string `json:"topics"` - Data string `json:"data"` + Address string `json:"address"` + Topics []string `json:"topics"` + Data string `json:"data"` } type rpcLogWithTxHash struct { @@ -171,7 +174,10 @@ func (p *EthereumParser) GetAddrDescFromAddress(address string) (bchain.AddressD if has0xPrefix(address) { address = address[2:] } - if len(address) == 0 { + if len(address) != EthereumTypeAddressDescriptorLen*2 { + if len(address) != 0 { + glog.Warning("Ignoring address ", address) + } return nil, bchain.ErrAddressMissing } if len(address)&1 == 1 { @@ -278,6 +284,10 @@ func (p *EthereumParser) PackTx(tx *bchain.Tx, height uint32, blockTime int64) ( } ptLogs := make([]*ProtoCompleteTransaction_ReceiptType_LogType, len(r.Receipt.Logs)) for i, l := range r.Receipt.Logs { + a, err := hexutil.Decode(l.Address) + if err != nil { + return nil, errors.Annotatef(err, "Address cannot be decoded %v", l) + } d, err := hexutil.Decode(l.Data) if err != nil { return nil, errors.Annotatef(err, "Data cannot be decoded %v", l) @@ -290,7 +300,7 @@ func (p *EthereumParser) PackTx(tx *bchain.Tx, height uint32, blockTime int64) ( } } ptLogs[i] = &ProtoCompleteTransaction_ReceiptType_LogType{ - Address: l.Address.Bytes(), + Address: a, Data: d, Topics: t, } @@ -332,7 +342,7 @@ func (p *EthereumParser) UnpackTx(buf []byte) (*bchain.Tx, uint32, error) { topics[j] = hexutil.Encode(t) } logs[i] = &rpcLog{ - Address: ethcommon.BytesToAddress(l.Address), + Address: hexutil.Encode(l.Address), Data: hexutil.Encode(l.Data), Topics: topics, } @@ -388,19 +398,39 @@ func (p *EthereumParser) GetChainType() bchain.ChainType { // GetHeightFromTx returns ethereum specific data from bchain.Tx func GetHeightFromTx(tx *bchain.Tx) (uint32, error) { - // TODO - temporary implementation - will use bchain.Tx.SpecificData field - b, err := hex.DecodeString(tx.Hex) - if err != nil { - return 0, err + var bn string + csd, ok := tx.CoinSpecificData.(completeTransaction) + if !ok { + b, err := hex.DecodeString(tx.Hex) + if err != nil { + return 0, err + } + var ct completeTransaction + err = json.Unmarshal(b, &ct) + if err != nil { + return 0, err + } + bn = ct.Tx.BlockNumber + } else { + bn = csd.Tx.BlockNumber } - var ct completeTransaction - var n uint64 - err = json.Unmarshal(b, &ct) + n, err := hexutil.DecodeUint64(bn) if err != nil { - return 0, err - } - if n, err = hexutil.DecodeUint64(ct.Tx.BlockNumber); err != nil { - return 0, errors.Annotatef(err, "BlockNumber %v", ct.Tx.BlockNumber) + return 0, errors.Annotatef(err, "BlockNumber %v", bn) } return uint32(n), nil } + +// GetErc20FromTx returns Erc20 data from bchain.Tx +func GetErc20FromTx(tx *bchain.Tx) ([]Erc20Transfer, error) { + var r []Erc20Transfer + var err error + csd, ok := tx.CoinSpecificData.(completeTransaction) + if ok { + r, err = erc20GetTransfersFromLog(csd.Receipt.Logs) + if err != nil { + return nil, err + } + } + return r, nil +} diff --git a/bchain/coins/eth/ethparser_test.go b/bchain/coins/eth/ethparser_test.go index 6294cb71..4ed3ec48 100644 --- a/bchain/coins/eth/ethparser_test.go +++ b/bchain/coins/eth/ethparser_test.go @@ -32,9 +32,10 @@ func TestEthParser_GetAddrDescFromAddress(t *testing.T) { want: "47526228d673e9f079630d6cdaff5a2ed13e0e60", }, { - name: "odd address", - args: args{address: "7526228d673e9f079630d6cdaff5a2ed13e0e60"}, - want: "07526228d673e9f079630d6cdaff5a2ed13e0e60", + name: "address of wrong length", + args: args{address: "7526228d673e9f079630d6cdaff5a2ed13e0e60"}, + want: "", + wantErr: true, }, { name: "ErrAddressMissing", @@ -51,7 +52,7 @@ func TestEthParser_GetAddrDescFromAddress(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - p := NewEthereumParser() + p := NewEthereumParser(1) got, err := p.GetAddrDescFromAddress(tt.args.address) if (err != nil) != tt.wantErr { t.Errorf("EthParser.GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) @@ -146,7 +147,7 @@ func TestEthereumParser_PackTx(t *testing.T) { want: testTxPacked2, }, } - p := NewEthereumParser() + p := NewEthereumParser(1) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := p.PackTx(tt.args.tx, tt.args.height, tt.args.blockTime) @@ -187,7 +188,7 @@ func TestEthereumParser_UnpackTx(t *testing.T) { want1: 2871048, }, } - p := NewEthereumParser() + p := NewEthereumParser(1) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { b, err := hex.DecodeString(tt.args.hex) diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 6339cf83..72513262 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -31,10 +31,11 @@ const ( // Configuration represents json config file type Configuration struct { - CoinName string `json:"coin_name"` - CoinShortcut string `json:"coin_shortcut"` - RPCURL string `json:"rpc_url"` - RPCTimeout int `json:"rpc_timeout"` + CoinName string `json:"coin_name"` + CoinShortcut string `json:"coin_shortcut"` + RPCURL string `json:"rpc_url"` + RPCTimeout int `json:"rpc_timeout"` + BlockAddressesToKeep int `json:"block_addresses_to_keep"` } // EthereumRPC is an interface to JSON-RPC eth service. @@ -65,6 +66,10 @@ func NewEthereumRPC(config json.RawMessage, pushHandler func(bchain.Notification if err != nil { return nil, errors.Annotatef(err, "Invalid configuration file") } + // keep at least 100 mappings block->addresses to allow rollback + if c.BlockAddressesToKeep < 100 { + c.BlockAddressesToKeep = 100 + } rc, err := rpc.Dial(c.RPCURL) if err != nil { return nil, err @@ -78,7 +83,7 @@ func NewEthereumRPC(config json.RawMessage, pushHandler func(bchain.Notification } // always create parser - s.Parser = NewEthereumParser() + s.Parser = NewEthereumParser(c.BlockAddressesToKeep) s.timeout = time.Duration(c.RPCTimeout) * time.Second // detect ethereum classic diff --git a/db/rocksdb.go b/db/rocksdb.go index 644e9592..e8169e31 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -59,13 +59,21 @@ const ( cfDefault = iota cfHeight cfAddresses - cfTxAddresses - cfAddressBalance cfBlockTxs cfTransactions + // BitcoinType + cfAddressBalance + cfTxAddresses + // EthereumType + cfAddressContracts = cfAddressBalance ) -var cfNames = []string{"default", "height", "addresses", "txAddresses", "addressBalance", "blockTxs", "transactions"} +// common columns +var cfNames = []string{"default", "height", "addresses", "blockTxs", "transactions"} + +// type specific columns +var cfNamesBitcoinType = []string{"addressBalance", "txAddresses"} +var cfNamesEthereumType = []string{"addressContracts"} func openDB(path string, c *gorocksdb.Cache, openFiles int) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { // opts with bloom filter @@ -73,9 +81,14 @@ func openDB(path string, c *gorocksdb.Cache, openFiles int) (*gorocksdb.DB, []*g // opts for addresses without bloom filter // from documentation: if most of your queries are executed using iterators, you shouldn't set bloom filter optsAddresses := createAndSetDBOptions(0, c, openFiles) - // default, height, addresses, txAddresses, addressBalance, blockTxids, transactions - fcOptions := []*gorocksdb.Options{opts, opts, optsAddresses, opts, opts, opts, opts} - db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions) + // default, height, addresses, blockTxids, transactions + cfOptions := []*gorocksdb.Options{opts, opts, optsAddresses, opts, opts} + // append type specific options + count := len(cfNames) - len(cfOptions) + for i := 0; i < count; i++ { + cfOptions = append(cfOptions, opts) + } + db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, cfOptions) if err != nil { return nil, nil, err } @@ -86,6 +99,16 @@ func openDB(path string, c *gorocksdb.Cache, openFiles int) (*gorocksdb.DB, []*g // needs to be called to release it. func NewRocksDB(path string, cacheSize, maxOpenFiles int, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) { glog.Infof("rocksdb: opening %s, required data version %v, cache size %v, max open files %v", path, dbVersion, cacheSize, maxOpenFiles) + + chainType := parser.GetChainType() + if chainType == bchain.ChainBitcoinType { + cfNames = append(cfNames, cfNamesBitcoinType...) + } else if chainType == bchain.ChainEthereumType { + cfNames = append(cfNames, cfNamesEthereumType...) + } else { + return nil, errors.New("Unknown chain type") + } + c := gorocksdb.NewLRUCache(cacheSize) db, cfh, err := openDB(path, c, maxOpenFiles) if err != nil { @@ -276,21 +299,18 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { if err := d.writeHeightFromBlock(wb, block, op); err != nil { return err } + addresses := make(map[string][]outpoint) if chainType == bchain.ChainBitcoinType { if op == opDelete { // block does not contain mapping tx-> input address, which is necessary to recreate // unspentTxs; therefore it is not possible to DisconnectBlocks this way return errors.New("DisconnectBlock is not supported for BitcoinType chains") } - addresses := make(map[string][]outpoint) txAddressesMap := make(map[string]*TxAddresses) balances := make(map[string]*AddrBalance) if err := d.processAddressesBitcoinType(block, addresses, txAddressesMap, balances); err != nil { return err } - if err := d.storeAddresses(wb, block.Height, addresses); err != nil { - return err - } if err := d.storeTxAddresses(wb, txAddressesMap); err != nil { return err } @@ -301,12 +321,23 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { return err } } else if chainType == bchain.ChainEthereumType { - if err := d.writeAddressesEthereumType(wb, block, op); err != nil { + addressContracts := make(map[string]*AddrContracts) + blockTxs, err := d.processAddressesEthereumType(block, addresses, addressContracts) + if err != nil { + return err + } + if err := d.storeAddressContracts(wb, addressContracts); err != nil { + return err + } + if err := d.storeAndCleanupBlockTxsEthereumType(wb, block, blockTxs); err != nil { return err } } else { return errors.New("Unknown chain type") } + if err := d.storeAddresses(wb, block.Height, addresses); err != nil { + return err + } return d.db.Write(d.wo, wb) } @@ -579,6 +610,26 @@ func (d *RocksDB) storeBalances(wb *gorocksdb.WriteBatch, abm map[string]*AddrBa return nil } +func (d *RocksDB) cleanupBlockTxs(wb *gorocksdb.WriteBatch, block *bchain.Block) error { + keep := d.chainParser.KeepBlockAddresses() + // cleanup old block address + if block.Height > uint32(keep) { + for rh := block.Height - uint32(keep); rh < block.Height; rh-- { + key := packUint(rh) + val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxs], key) + if err != nil { + return err + } + if val.Size() == 0 { + break + } + val.Free() + d.db.DeleteCF(d.wo, d.cfh[cfBlockTxs], key) + } + } + return nil +} + func (d *RocksDB) storeAndCleanupBlockTxs(wb *gorocksdb.WriteBatch, block *bchain.Block) error { pl := d.chainParser.PackedTxidLen() buf := make([]byte, 0, pl*len(block.Txs)) @@ -612,23 +663,7 @@ func (d *RocksDB) storeAndCleanupBlockTxs(wb *gorocksdb.WriteBatch, block *bchai } key := packUint(block.Height) wb.PutCF(d.cfh[cfBlockTxs], key, buf) - keep := d.chainParser.KeepBlockAddresses() - // cleanup old block address - if block.Height > uint32(keep) { - for rh := block.Height - uint32(keep); rh < block.Height; rh-- { - key = packUint(rh) - val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxs], key) - if err != nil { - return err - } - if val.Size() == 0 { - break - } - val.Free() - d.db.DeleteCF(d.wo, d.cfh[cfBlockTxs], key) - } - } - return nil + return d.cleanupBlockTxs(wb, block) } func (d *RocksDB) getBlockTxs(height uint32) ([]blockTxs, error) { @@ -844,74 +879,6 @@ func (d *RocksDB) unpackNOutpoints(buf []byte) ([]outpoint, int, error) { return outpoints, p, nil } -func (d *RocksDB) addAddrDescToRecords(op int, wb *gorocksdb.WriteBatch, records map[string][]outpoint, addrDesc bchain.AddressDescriptor, btxid []byte, vout int32, bh uint32) error { - if len(addrDesc) > 0 { - if len(addrDesc) > maxAddrDescLen { - glog.Infof("rocksdb: block %d, skipping addrDesc of length %d", bh, len(addrDesc)) - } else { - strAddrDesc := string(addrDesc) - records[strAddrDesc] = append(records[strAddrDesc], outpoint{ - btxID: btxid, - index: vout, - }) - if op == opDelete { - // remove transactions from cache - d.internalDeleteTx(wb, btxid) - } - } - } - return nil -} - -func (d *RocksDB) writeAddressesEthereumType(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { - addresses := make(map[string][]outpoint) - for _, tx := range block.Txs { - btxID, err := d.chainParser.PackTxid(tx.Txid) - if err != nil { - return err - } - for _, output := range tx.Vout { - addrDesc, err := d.chainParser.GetAddrDescFromVout(&output) - if err != nil { - // do not log ErrAddressMissing, transactions can be without to address (for example eth contracts) - if err != bchain.ErrAddressMissing { - glog.Warningf("rocksdb: addrDesc: %v - height %d, tx %v, output %v", err, block.Height, tx.Txid, output) - } - continue - } - err = d.addAddrDescToRecords(op, wb, addresses, addrDesc, btxID, int32(output.N), block.Height) - if err != nil { - return err - } - } - // store inputs in format txid ^index - for _, input := range tx.Vin { - for i, a := range input.Addresses { - addrDesc, err := d.chainParser.GetAddrDescFromAddress(a) - if err != nil { - glog.Warningf("rocksdb: addrDesc: %v - %d %s", err, block.Height, addrDesc) - continue - } - err = d.addAddrDescToRecords(op, wb, addresses, addrDesc, btxID, int32(^i), block.Height) - if err != nil { - return err - } - } - } - } - for addrDesc, outpoints := range addresses { - key := packAddressKey(bchain.AddressDescriptor(addrDesc), block.Height) - switch op { - case opInsert: - val := d.packOutpoints(outpoints) - wb.PutCF(d.cfh[cfAddresses], key, val) - case opDelete: - wb.DeleteCF(d.cfh[cfAddresses], key) - } - } - return nil -} - // Block index // BlockInfo holds information about blocks kept in column height @@ -1229,37 +1196,6 @@ func (d *RocksDB) DisconnectBlockRangeBitcoinType(lower uint32, higher uint32) e return err } -// DisconnectBlockRangeNonUTXO performs full range scan to remove a range of blocks -// it is very slow operation -func (d *RocksDB) DisconnectBlockRangeNonUTXO(lower uint32, higher uint32) error { - glog.Infof("db: disconnecting blocks %d-%d", lower, higher) - addrKeys, _, err := d.allAddressesScan(lower, higher) - if err != nil { - return err - } - glog.Infof("rocksdb: about to disconnect %d addresses ", len(addrKeys)) - wb := gorocksdb.NewWriteBatch() - defer wb.Destroy() - for _, addrKey := range addrKeys { - if glog.V(2) { - glog.Info("address ", hex.EncodeToString(addrKey)) - } - // delete address:height from the index - wb.DeleteCF(d.cfh[cfAddresses], addrKey) - } - for height := lower; height <= higher; height++ { - if glog.V(2) { - glog.Info("height ", height) - } - wb.DeleteCF(d.cfh[cfHeight], packUint(height)) - } - err = d.db.Write(d.wo, wb) - if err == nil { - glog.Infof("rocksdb: blocks %d-%d disconnected", lower, higher) - } - return err -} - func dirSize(path string) (int64, error) { var size int64 err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { diff --git a/db/rocksdb_ethereumtype.go b/db/rocksdb_ethereumtype.go new file mode 100644 index 00000000..96033c83 --- /dev/null +++ b/db/rocksdb_ethereumtype.go @@ -0,0 +1,272 @@ +package db + +import ( + "blockbook/bchain" + "blockbook/bchain/coins/eth" + "bytes" + "encoding/hex" + + "github.com/bsm/go-vlq" + "github.com/golang/glog" + "github.com/juju/errors" + "github.com/tecbot/gorocksdb" +) + +// AddrContract is Contract address with number of transactions done by given address +type AddrContract struct { + Contract bchain.AddressDescriptor + Txs uint +} + +// AddrContracts is array of contracts with +type AddrContracts struct { + EthTxs uint + Contracts []AddrContract +} + +func (d *RocksDB) storeAddressContracts(wb *gorocksdb.WriteBatch, acm map[string]*AddrContracts) error { + buf := make([]byte, 64) + varBuf := make([]byte, vlq.MaxLen64) + for addrDesc, acs := range acm { + // address with 0 contracts is removed from db - happens on disconnect + if acs == nil || (acs.EthTxs == 0 && len(acs.Contracts) == 0) { + wb.DeleteCF(d.cfh[cfAddressContracts], bchain.AddressDescriptor(addrDesc)) + } else { + buf = buf[:0] + l := packVaruint(acs.EthTxs, varBuf) + buf = append(buf, varBuf[:l]...) + for _, ac := range acs.Contracts { + buf = append(buf, ac.Contract...) + l = packVaruint(ac.Txs, varBuf) + buf = append(buf, varBuf[:l]...) + } + wb.PutCF(d.cfh[cfAddressContracts], bchain.AddressDescriptor(addrDesc), buf) + } + } + return nil +} + +// GetAddrDescContracts returns AddrContracts for given addrDesc +func (d *RocksDB) GetAddrDescContracts(addrDesc bchain.AddressDescriptor) (*AddrContracts, error) { + val, err := d.db.GetCF(d.ro, d.cfh[cfAddressContracts], addrDesc) + if err != nil { + return nil, err + } + defer val.Free() + buf := val.Data() + if len(buf) == 0 { + return nil, nil + } + c := make([]AddrContract, 0, 4) + et, l := unpackVaruint(buf) + buf = buf[l:] + for len(buf) > 0 { + if len(buf) < eth.EthereumTypeAddressDescriptorLen { + return nil, errors.New("Invalid data stored in cfAddressContracts for AddrDesc " + addrDesc.String()) + } + txs, l := unpackVaruint(buf[eth.EthereumTypeAddressDescriptorLen:]) + contract := make(bchain.AddressDescriptor, eth.EthereumTypeAddressDescriptorLen) + copy(contract, buf[:eth.EthereumTypeAddressDescriptorLen]) + c = append(c, AddrContract{ + Contract: contract, + Txs: txs, + }) + buf = buf[eth.EthereumTypeAddressDescriptorLen+l:] + } + return &AddrContracts{ + EthTxs: et, + Contracts: c}, nil +} + +func (d *RocksDB) addToAddressesAndContractsEthereumType(addrDesc bchain.AddressDescriptor, btxID []byte, index int32, contract bchain.AddressDescriptor, addresses map[string][]outpoint, addressContracts map[string]*AddrContracts) error { + var err error + strAddrDesc := string(addrDesc) + ac, e := addressContracts[strAddrDesc] + if !e { + ac, err = d.GetAddrDescContracts(addrDesc) + if err != nil { + return err + } + if ac == nil { + ac = &AddrContracts{} + } + addressContracts[strAddrDesc] = ac + d.cbs.balancesMiss++ + } else { + d.cbs.balancesHit++ + } + if contract == nil { + ac.EthTxs++ + } else { + // locate the contract and set i to the index in the array of contracts + var i int + var found bool + for i = range ac.Contracts { + if bytes.Equal(contract, ac.Contracts[i].Contract) { + found = true + break + } + } + if !found { + i = len(ac.Contracts) + ac.Contracts = append(ac.Contracts, AddrContract{Contract: contract}) + } + // index 0 is for ETH transfers, contract indexes start with 1 + if index < 0 { + index = ^int32(i + 1) + } else { + index = int32(i + 1) + } + ac.Contracts[i].Txs++ + } + addresses[strAddrDesc] = append(addresses[strAddrDesc], outpoint{ + btxID: btxID, + index: index, + }) + return nil +} + +type ethBlockTxContract struct { + addr, contract bchain.AddressDescriptor +} + +type ethBlockTx struct { + btxID []byte + from, to bchain.AddressDescriptor + contracts []ethBlockTxContract +} + +func (d *RocksDB) processAddressesEthereumType(block *bchain.Block, addresses map[string][]outpoint, addressContracts map[string]*AddrContracts) ([]ethBlockTx, error) { + blockTxs := make([]ethBlockTx, len(block.Txs)) + for txi, tx := range block.Txs { + btxID, err := d.chainParser.PackTxid(tx.Txid) + if err != nil { + return nil, err + } + blockTx := &blockTxs[txi] + blockTx.btxID = btxID + // there is only one output address in EthereumType transaction, store it in format txid 0 + if len(tx.Vout) == 1 && len(tx.Vout[0].ScriptPubKey.Addresses) == 1 { + addrDesc, err := d.chainParser.GetAddrDescFromAddress(tx.Vout[0].ScriptPubKey.Addresses[0]) + if err != nil { + // do not log ErrAddressMissing, transactions can be without to address (for example eth contracts) + if err != bchain.ErrAddressMissing { + glog.Warningf("rocksdb: addrDesc: %v - height %d, tx %v, output", err, block.Height, tx.Txid) + } + continue + } + if err = d.addToAddressesAndContractsEthereumType(addrDesc, btxID, 0, nil, addresses, addressContracts); err != nil { + return nil, err + } + blockTx.to = addrDesc + } + // there is only one input address in EthereumType transaction, store it in format txid ^0 + if len(tx.Vin) == 1 && len(tx.Vin[0].Addresses) == 1 { + addrDesc, err := d.chainParser.GetAddrDescFromAddress(tx.Vin[0].Addresses[0]) + if err != nil { + if err != bchain.ErrAddressMissing { + glog.Warningf("rocksdb: addrDesc: %v - height %d, tx %v, input", err, block.Height, tx.Txid) + } + continue + } + if err = d.addToAddressesAndContractsEthereumType(addrDesc, btxID, ^int32(0), nil, addresses, addressContracts); err != nil { + return nil, err + } + blockTx.from = addrDesc + } + // store erc20 transfers + erc20, err := eth.GetErc20FromTx(&tx) + if err != nil { + glog.Warningf("rocksdb: GetErc20FromTx %v - height %d, tx %v", err, block.Height, tx.Txid) + } + blockTx.contracts = make([]ethBlockTxContract, len(erc20)*2) + for i, t := range erc20 { + var contract, from, to bchain.AddressDescriptor + contract, err = d.chainParser.GetAddrDescFromAddress(t.Contract) + if err == nil { + from, err = d.chainParser.GetAddrDescFromAddress(t.From) + if err == nil { + to, err = d.chainParser.GetAddrDescFromAddress(t.To) + } + } + if err != nil { + glog.Warningf("rocksdb: GetErc20FromTx %v - height %d, tx %v, transfer %v", err, block.Height, tx.Txid, t) + continue + } + if err = d.addToAddressesAndContractsEthereumType(from, btxID, ^int32(i), contract, addresses, addressContracts); err != nil { + return nil, err + } + bc := &blockTx.contracts[i*2] + bc.addr = from + bc.contract = contract + if err = d.addToAddressesAndContractsEthereumType(to, btxID, int32(i), contract, addresses, addressContracts); err != nil { + return nil, err + } + bc = &blockTx.contracts[i*2+1] + bc.addr = to + bc.contract = contract + } + } + return blockTxs, nil +} + +func (d *RocksDB) storeAndCleanupBlockTxsEthereumType(wb *gorocksdb.WriteBatch, block *bchain.Block, blockTxs []ethBlockTx) error { + pl := d.chainParser.PackedTxidLen() + buf := make([]byte, 0, (pl+2*eth.EthereumTypeAddressDescriptorLen)*len(blockTxs)) + varBuf := make([]byte, vlq.MaxLen64) + zeroAddress := make([]byte, eth.EthereumTypeAddressDescriptorLen) + appendAddress := func(a bchain.AddressDescriptor) { + if len(a) != eth.EthereumTypeAddressDescriptorLen { + buf = append(buf, zeroAddress...) + } else { + buf = append(buf, a...) + } + } + for i := range blockTxs { + blockTx := &blockTxs[i] + buf = append(buf, blockTx.btxID...) + appendAddress(blockTx.from) + appendAddress(blockTx.to) + l := packVaruint(uint(len(blockTx.contracts)), varBuf) + buf = append(buf, varBuf[:l]...) + for j := range blockTx.contracts { + c := &blockTx.contracts[j] + appendAddress(c.addr) + appendAddress(c.contract) + } + } + key := packUint(block.Height) + wb.PutCF(d.cfh[cfBlockTxs], key, buf) + return d.cleanupBlockTxs(wb, block) +} + +// DisconnectBlockRangeNonUTXO performs full range scan to remove a range of blocks +// it is very slow operation +func (d *RocksDB) DisconnectBlockRangeNonUTXO(lower uint32, higher uint32) error { + glog.Infof("db: disconnecting blocks %d-%d", lower, higher) + addrKeys, _, err := d.allAddressesScan(lower, higher) + if err != nil { + return err + } + glog.Infof("rocksdb: about to disconnect %d addresses ", len(addrKeys)) + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + for _, addrKey := range addrKeys { + if glog.V(2) { + glog.Info("address ", hex.EncodeToString(addrKey)) + } + // delete address:height from the index + wb.DeleteCF(d.cfh[cfAddresses], addrKey) + } + for height := lower; height <= higher; height++ { + if glog.V(2) { + glog.Info("height ", height) + } + wb.DeleteCF(d.cfh[cfHeight], packUint(height)) + } + err = d.db.Write(d.wo, wb) + if err == nil { + glog.Infof("rocksdb: blocks %d-%d disconnected", lower, higher) + } + return err +} diff --git a/db/rocksdb_ethereumtype_test.go b/db/rocksdb_ethereumtype_test.go new file mode 100644 index 00000000..2b05c46d --- /dev/null +++ b/db/rocksdb_ethereumtype_test.go @@ -0,0 +1,183 @@ +// build unittest + +package db + +import ( + "blockbook/bchain/coins/eth" + "blockbook/tests/dbtestdata" + "testing" +) + +type testEthereumParser struct { + *eth.EthereumParser +} + +func ethereumTestnetParser() *eth.EthereumParser { + return eth.NewEthereumParser(1) +} + +func verifyAfterEthereumTypeBlock1(t *testing.T, d *RocksDB, afterDisconnect bool) { + if err := checkColumn(d, cfHeight, []keyPair{ + keyPair{ + "0041eee8", + "c7b98df95acfd11c51ba25611a39e004fe56c8fdfc1582af99354fcd09c17b11" + uintToHex(1534858022) + varuintToHex(2) + varuintToHex(31839), + nil, + }, + }); err != nil { + { + t.Fatal(err) + } + } + // the vout is encoded as signed varint, i.e. value * 2 for positive values, abs(value)*2 + 1 for negative values + if err := checkColumn(d, cfAddresses, []keyPair{ + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "01", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "00" + dbtestdata.EthTxidB1T2 + "02", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "01" + dbtestdata.EthTxidB1T2 + "03", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "00", nil}, + }); err != nil { + { + t.Fatal(err) + } + } + + if err := checkColumn(d, cfAddressContracts, []keyPair{ + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser), "01", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser), "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser), "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser), "01", nil}, + }); err != nil { + { + t.Fatal(err) + } + } + + var blockTxsKp []keyPair + if afterDisconnect { + blockTxsKp = []keyPair{} + } else { + blockTxsKp = []keyPair{ + keyPair{ + "0041eee8", + dbtestdata.EthTxidB1T1 + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "00" + + dbtestdata.EthTxidB1T2 + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + + "02" + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser), + nil, + }, + } + } + if err := checkColumn(d, cfBlockTxs, blockTxsKp); err != nil { + { + t.Fatal(err) + } + } +} + +func verifyAfterEthereumTypeBlock2(t *testing.T, d *RocksDB) { + if err := checkColumn(d, cfHeight, []keyPair{ + keyPair{ + "0041eee8", + "c7b98df95acfd11c51ba25611a39e004fe56c8fdfc1582af99354fcd09c17b11" + uintToHex(1534858022) + varuintToHex(2) + varuintToHex(31839), + nil, + }, + keyPair{ + "0041eee9", + "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6" + uintToHex(1534859988) + varuintToHex(2) + varuintToHex(2345678), + nil, + }, + }); err != nil { + { + t.Fatal(err) + } + } + // the vout is encoded as signed varint, i.e. value * 2 for positive values, abs(value)*2 + 1 for negative values + if err := checkColumn(d, cfAddresses, []keyPair{ + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "01", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "00" + dbtestdata.EthTxidB1T2 + "02", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "01" + dbtestdata.EthTxidB1T2 + "03", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "00", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T1 + "01" + dbtestdata.EthTxidB2T2 + "05" + dbtestdata.EthTxidB2T2 + "02", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr9f, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T1 + "00", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T2 + "01" + dbtestdata.EthTxidB2T2 + "02" + dbtestdata.EthTxidB2T2 + "05" + dbtestdata.EthTxidB2T2 + "04" + dbtestdata.EthTxidB2T2 + "03", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr7b, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T2 + "03" + dbtestdata.EthTxidB2T2 + "04", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract47, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T2 + "00", nil}, + }); err != nil { + { + t.Fatal(err) + } + } + + if err := checkColumn(d, cfAddressContracts, []keyPair{ + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser), "01", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser), "02" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "02" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + "01", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser), "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser), "01", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr9f, d.chainParser), "01", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser), "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + "02" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "02", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr7b, d.chainParser), "00" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + "01", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract47, d.chainParser), "01", nil}, + }); err != nil { + { + t.Fatal(err) + } + } + + if err := checkColumn(d, cfBlockTxs, []keyPair{ + keyPair{ + "0041eee9", + dbtestdata.EthTxidB2T1 + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr9f, d.chainParser) + "00" + + dbtestdata.EthTxidB2T2 + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract47, d.chainParser) + + "08" + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr7b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr7b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser), + nil, + }, + }); err != nil { + { + t.Fatal(err) + } + } +} + +// TestRocksDB_Index_EthereumType is an integration test probing the whole indexing functionality for EthereumType chains +// It does the following: +// 1) Connect two blocks (inputs from 2nd block are spending some outputs from the 1st block) +// 2) GetTransactions for various addresses / low-high ranges +// 3) GetBestBlock, GetBlockHash +// 4) Test tx caching functionality +// 5) Disconnect block 2 - expect error +// 6) Disconnect the block 2 using BlockTxs column +// 7) Reconnect block 2 and check +// After each step, the content of DB is examined and any difference against expected state is regarded as failure +func TestRocksDB_Index_EthereumType(t *testing.T) { + d := setupRocksDB(t, &testEthereumParser{ + EthereumParser: ethereumTestnetParser(), + }) + defer closeAndDestroyRocksDB(t, d) + + // connect 1st block - will log warnings about missing UTXO transactions in txAddresses column + block1 := dbtestdata.GetTestEthereumTypeBlock1(d.chainParser) + if err := d.ConnectBlock(block1); err != nil { + t.Fatal(err) + } + verifyAfterEthereumTypeBlock1(t, d, false) + + // connect 2nd block - use some outputs from the 1st block as the inputs and 1 input uses tx from the same block + block2 := dbtestdata.GetTestEthereumTypeBlock2(d.chainParser) + if err := d.ConnectBlock(block2); err != nil { + t.Fatal(err) + } + verifyAfterEthereumTypeBlock2(t, d) + +} diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 4d89b6b9..fe8d50af 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -33,6 +33,10 @@ func TestMain(m *testing.M) { os.Exit(c) } +type testBitcoinParser struct { + *btc.BitcoinParser +} + func bitcoinTestnetParser() *btc.BitcoinParser { return btc.NewBitcoinParser( btc.GetChainParams("test"), @@ -48,7 +52,7 @@ func setupRocksDB(t *testing.T, p bchain.BlockChainParser) *RocksDB { if err != nil { t.Fatal(err) } - is, err := d.LoadInternalState("btc-testnet") + is, err := d.LoadInternalState("coin-unittest") if err != nil { t.Fatal(err) } @@ -151,7 +155,7 @@ func checkColumn(d *RocksDB, col int, kp []keyPair) error { return nil } -func verifyAfterBitcoinTypeOBlock1(t *testing.T, d *RocksDB, afterDisconnect bool) { +func verifyAfterBitcoinTypeBlock1(t *testing.T, d *RocksDB, afterDisconnect bool) { if err := checkColumn(d, cfHeight, []keyPair{ keyPair{ "000370d5", @@ -388,10 +392,6 @@ func verifyGetTransactions(t *testing.T, d *RocksDB, addr string, low, high uint } } -type testBitcoinParser struct { - *btc.BitcoinParser -} - // override PackTx and UnpackTx to default BaseParser functionality // BitcoinParser uses tx hex which is not available for the test transactions func (p *testBitcoinParser) PackTx(tx *bchain.Tx, height uint32, blockTime int64) ([]byte, error) { @@ -444,7 +444,7 @@ func TestRocksDB_Index_BitcoinType(t *testing.T) { if err := d.ConnectBlock(block1); err != nil { t.Fatal(err) } - verifyAfterBitcoinTypeOBlock1(t, d, false) + verifyAfterBitcoinTypeBlock1(t, d, false) // connect 2nd block - use some outputs from the 1st block as the inputs and 1 input uses tx from the same block block2 := dbtestdata.GetTestBitcoinTypeBlock2(d.chainParser) @@ -552,7 +552,7 @@ func TestRocksDB_Index_BitcoinType(t *testing.T) { if err != nil { t.Fatal(err) } - verifyAfterBitcoinTypeOBlock1(t, d, true) + verifyAfterBitcoinTypeBlock1(t, d, true) if err := checkColumn(d, cfTransactions, []keyPair{}); err != nil { { t.Fatal(err) diff --git a/tests/dbtestdata/dbtestdata_ethereumtype.go b/tests/dbtestdata/dbtestdata_ethereumtype.go new file mode 100644 index 00000000..4f228182 --- /dev/null +++ b/tests/dbtestdata/dbtestdata_ethereumtype.go @@ -0,0 +1,69 @@ +package dbtestdata + +import ( + "blockbook/bchain" + "encoding/hex" +) + +const ( + EthAddr3e = "3e3a3d69dc66ba10737f531ed088954a9ec89d97" + EthAddr55 = "555ee11fbddc0e49a9bab358a8941ad95ffdb48f" + EthAddr20 = "20cd153de35d469ba46127a0c8f18626b59a256a" + EthAddr9f = "9f4981531fda132e83c44680787dfa7ee31e4f8d" + EthAddr4b = "4bda106325c335df99eab7fe363cac8a0ba2a24d" + EthAddr7b = "7b62eb7fe80350dc7ec945c0b73242cb9877fb1b" + EthAddrContract4a = "4af4114f73d1c1c903ac9e0361b379d1291808a2" // ERC-20 (VTY) + EthAddrContract0d = "0d0f936ee4c93e25944694d6c121de94d9760f11" // ERC-20 (MTT) + EthAddrContract47 = "479cc461fecd078f766ecc58533d6f69580cf3ac" // non ERC20 + + EthTxidB1T1 = "cd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b" + EthTx1Packed = "08889eaf0110fa83c3d5051a6908ece40212050430e234001888a40122081bc0159d530e60003220cd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b3a14555ee11fbddc0e49a9bab358a8941ad95ffdb48f42143e3a3d69dc66ba10737f531ed088954a9ec89d97480a22070a025208120101" + EthTxidB1T2 = "a9cd088aba2131000da6f38a33c20169baee476218deea6b78720700b895b101" + EthTx2Packed = "08d38388021092f4c1d5051aa20108d001120509502f900018d5e1042a44a9059cbb000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f00000000000000000000000000000000000000000000021e19e0c9bab24000003220a9cd088aba2131000da6f38a33c20169baee476218deea6b78720700b895b1013a144af4114f73d1c1c903ac9e0361b379d1291808a2421420cd153de35d469ba46127a0c8f18626b59a256a22a8010a02cb391201011a9e010a144af4114f73d1c1c903ac9e0361b379d1291808a2122000000000000000000000000000000000000000000000021e19e0c9bab24000001a20ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef1a2000000000000000000000000020cd153de35d469ba46127a0c8f18626b59a256a1a20000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f" + EthTxidB2T1 = "c2c3dd1ecb00e8a6d81f793d24387cf2947a313e94ab03b1fb22cd63320f6c91" + EthTx3Packed = "08fda28c0210d8eb95df051a6708c20112050218711a001888a401220710bc3578bd37d83220c2c3dd1ecb00e8a6d81f793d24387cf2947a313e94ab03b1fb22cd63320f6c913a149f4981531fda132e83c44680787dfa7ee31e4f8d4214555ee11fbddc0e49a9bab358a8941ad95ffdb48f480722070a025208120101" + EthTxidB2T2 = "c92919ad24ffd58f760b18df7949f06e1190cf54a50a0e3745a385608ed3cbf2" + EthTx4Packed = "088795890210b5e1ebde051aa50b08f6be0712043b9aca001890a10f2ac40a4f15078700000000000000000000000000000000000000000000000000000000000000c0000000000000000000000000000000000000000000000000000000000000022000000000000000000000000000000000000000000000000000000000000003c00000000000000000000000000000000000000000000000000000000000000420000000000000000000000000000000000000000000000000000000000000048000000000000000000000000000000000000000000000000000000000000004e00000000000000000000000000000000000000000000000000000000000000002000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f0000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d0000000000000000000000000d0f936ee4c93e25944694d6c121de94d9760f110000000000000000000000004af4114f73d1c1c903ac9e0361b379d1291808a200000000000000000000000000000000000000000000000000000000000000000000000000000000000000007b62eb7fe80350dc7ec945c0b73242cb9877fb1b0000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d0000000000000000000000004af4114f73d1c1c903ac9e0361b379d1291808a20000000000000000000000000d0f936ee4c93e25944694d6c121de94d9760f110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000a5ef5a7656bfb0000000000000000000000000000000000000000000000000000004ba78398d5c5000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000166cfe0b9579b4ecf7a2801880f644009a324671a79754ea57c3a103c6e70d3dbef6ba69a08000000000000000000000000000000000000000000000000004f937d86afb90000000000000000000000000000000000000000000000000ab280fd8037d500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000166cfb784b7c1f3fbe8b75484603ab8adc58aaee3a46245a6579fac7077b5570018b4e0d4eb0000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000308fd0e798ac00000000000000000000000000000000000000000000000006a8313d60b1f80000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000001b000000000000000000000000000000000000000000000000000000000000001b00000000000000000000000000000000000000000000000000000000000000029de0ccec59e8948e3d905b40e5542335ebc1eb4674db517d2f6392ec7fdeb3d45f3449d313ee2589819c6c79eb1c1b047adae68565c1608e3a1d1d70823febb0000000000000000000000000000000000000000000000000000000000000000234d06fe17f1202e8b07177a30eb64d14adc08cdb3fa1b3e3e0bea0f9672c02175b77c01c51d3c7e460723b27ecbc7801fd6482559a8c9999593f9a4d149c73843220c92919ad24ffd58f760b18df7949f06e1190cf54a50a0e3745a385608ed3cbf23a14479cc461fecd078f766ecc58533d6f69580cf3ac42144bda106325c335df99eab7fe363cac8a0ba2a24d482422d40b0a03034d301201011a9e010a140d0f936ee4c93e25944694d6c121de94d9760f1112200000000000000000000000000000000000000000000000006a8313d60b1f606b1a20ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef1a20000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f1a200000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d1a9e010a144af4114f73d1c1c903ac9e0361b379d1291808a21220000000000000000000000000000000000000000000000000000308fd0e798ac01a20ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef1a200000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d1a20000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f1aa1030a14479cc461fecd078f766ecc58533d6f69580cf3ac1280020000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d0000000000000000000000000d0f936ee4c93e25944694d6c121de94d9760f110000000000000000000000004af4114f73d1c1c903ac9e0361b379d1291808a20000000000000000000000000000000000000000000000006a8313d60b1f606b000000000000000000000000000000000000000000000000000308fd0e798ac0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005e083a16f4b092c5729a49f9c3ed3cc171bb3d3d0c22e20b1de6063c32f399ac1a200d0b9391970d9a25552f37d436d2aae2925e2bfe1b2a923754bada030c498cb31a20000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f1a2000000000000000000000000000000000000000000000000000000000000000001a205af266c0a89a07c1917deaa024414577e6c3c31c8907d079e13eb448c082594f1a9e010a144af4114f73d1c1c903ac9e0361b379d1291808a2122000000000000000000000000000000000000000000000000000031855667df7a81a20ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef1a200000000000000000000000007b62eb7fe80350dc7ec945c0b73242cb9877fb1b1a200000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d1a9e010a140d0f936ee4c93e25944694d6c121de94d9760f1112200000000000000000000000000000000000000000000000006a8313d60b1f80001a20ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef1a200000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d1a200000000000000000000000007b62eb7fe80350dc7ec945c0b73242cb9877fb1b1aa1030a14479cc461fecd078f766ecc58533d6f69580cf3ac1280020000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d0000000000000000000000004af4114f73d1c1c903ac9e0361b379d1291808a20000000000000000000000000d0f936ee4c93e25944694d6c121de94d9760f1100000000000000000000000000000000000000000000000000031855667df7a80000000000000000000000000000000000000000000000006a8313d60b1f800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f2b0d62c44ed08f2a5adef40c875d20310a42a9d4f488bd26323256fe01c7f481a200d0b9391970d9a25552f37d436d2aae2925e2bfe1b2a923754bada030c498cb31a200000000000000000000000007b62eb7fe80350dc7ec945c0b73242cb9877fb1b1a2000000000000000000000000000000000000000000000000000000000000000001a20b0b69dad58df6032c3b266e19b1045b19c87acd2c06fb0c598090f44b8e263aa" +) + +func unpackTxs(packed []string, parser bchain.BlockChainParser) []bchain.Tx { + r := make([]bchain.Tx, len(packed)) + for i, p := range packed { + b, err := hex.DecodeString(p) + if err != nil { + panic(err) + } + tx, _, err := parser.UnpackTx(b) + if err != nil { + panic(err) + } + r[i] = *tx + } + return r +} + +func GetTestEthereumTypeBlock1(parser bchain.BlockChainParser) *bchain.Block { + return &bchain.Block{ + BlockHeader: bchain.BlockHeader{ + Height: 4321000, + Hash: "0xc7b98df95acfd11c51ba25611a39e004fe56c8fdfc1582af99354fcd09c17b11", + Size: 31839, + Time: 1534858022, + Confirmations: 2, + }, + Txs: unpackTxs([]string{EthTx1Packed, EthTx2Packed}, parser), + } +} + +func GetTestEthereumTypeBlock2(parser bchain.BlockChainParser) *bchain.Block { + return &bchain.Block{ + BlockHeader: bchain.BlockHeader{ + Height: 4321001, + Hash: "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6", + Size: 2345678, + Time: 1534859988, + Confirmations: 1, + }, + Txs: unpackTxs([]string{EthTx3Packed, EthTx4Packed}, parser), + } +}