Implement index v3 for ethereum type coin - WIP

bchain/baseparser.go

@@ -50,18 +50,18 @@ func (p *BaseParser) AmountToBigInt(n json.Number) (big.Int, error) {
 	return r, nil
 }
 
-// AmountToDecimalString converts amount in big.Int to string with decimal point in the correct place
+// AmountToDecimalString converts amount in big.Int to string with decimal point in the place defined by the parameter d
-func (p *BaseParser) AmountToDecimalString(a *big.Int) string {
+func AmountToDecimalString(a *big.Int, d int) string {
 	n := a.String()
 	var s string
 	if n[0] == '-' {
 		n = n[1:]
 		s = "-"
 	}
-	if len(n) <= p.AmountDecimalPoint {
+	if len(n) <= d {
-		n = zeros[:p.AmountDecimalPoint-len(n)+1] + n
+		n = zeros[:d-len(n)+1] + n
 	}
-	i := len(n) - p.AmountDecimalPoint
+	i := len(n) - d
 	ad := strings.TrimRight(n[i:], "0")
 	if len(ad) > 0 {
 		n = n[:i] + "." + ad
@@ -71,6 +71,11 @@ func (p *BaseParser) AmountToDecimalString(a *big.Int) string {
 	return s + n
 }
 
+// AmountToDecimalString converts amount in big.Int to string with decimal point in the correct place
+func (p *BaseParser) AmountToDecimalString(a *big.Int) string {
+	return AmountToDecimalString(a, p.AmountDecimalPoint)
+}
+
 // ParseTxFromJson parses JSON message containing transaction and returns Tx struct
 func (p *BaseParser) ParseTxFromJson(msg json.RawMessage) (*Tx, error) {
 	var tx Tx

bchain/coins/eth/erc20.go

@@ -25,25 +25,26 @@ var erc20abi = `[{"constant":true,"inputs":[],"name":"name","outputs":[{"name":"
 // doing the parsing/processing without using go-ethereum/accounts/abi library, it is simple to get data from Transfer event
 const erc20EventTransferSignature = "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"
 
-type erc20Transfer struct {
+// Erc20Transfer contains a single Erc20 token transfer
-	Contract ethcommon.Address
+type Erc20Transfer struct {
-	From     ethcommon.Address
+	Contract string
-	To       ethcommon.Address
+	From     string
+	To       string
 	Tokens   big.Int
 }
 
-func addressFromPaddedHex(s string) (*ethcommon.Address, error) {
+func addressFromPaddedHex(s string) (string, error) {
 	var t big.Int
 	_, ok := t.SetString(s, 0)
 	if !ok {
-		return nil, errors.New("Data is not a number")
+		return "", errors.New("Data is not a number")
 	}
 	a := ethcommon.BigToAddress(&t)
-	return &a, nil
+	return a.String(), nil
 }
 
-func erc20GetTransfersFromLog(logs []*rpcLog) ([]erc20Transfer, error) {
+func erc20GetTransfersFromLog(logs []*rpcLog) ([]Erc20Transfer, error) {
-	var r []erc20Transfer
+	var r []Erc20Transfer
 	for _, l := range logs {
 		if len(l.Topics) == 3 && l.Topics[0] == erc20EventTransferSignature {
 			var t big.Int
@@ -59,10 +60,10 @@ func erc20GetTransfersFromLog(logs []*rpcLog) ([]erc20Transfer, error) {
 			if err != nil {
 				return nil, err
 			}
-			r = append(r, erc20Transfer{
+			r = append(r, Erc20Transfer{
 				Contract: l.Address,
-				From:     *from,
+				From:     from,
-				To:       *to,
+				To:       to,
 				Tokens:   t,
 			})
 		}

bchain/coins/eth/erc20_test.go

@@ -3,25 +3,24 @@
 package eth
 
 import (
+	fmt "fmt"
 	"math/big"
-	"reflect"
+	"strings"
 	"testing"
-
-	ethcommon "github.com/ethereum/go-ethereum/common"
 )
 
 func TestErc20_erc20GetTransfersFromLog(t *testing.T) {
 	tests := []struct {
 		name    string
 		args    []*rpcLog
-		want    []erc20Transfer
+		want    []Erc20Transfer
 		wantErr bool
 	}{
 		{
 			name: "1",
 			args: []*rpcLog{
 				&rpcLog{
-					Address: ethcommon.HexToAddress("0x76a45e8976499ab9ae223cc584019341d5a84e96"),
+					Address: "0x76a45e8976499ab9ae223cc584019341d5a84e96",
 					Topics: []string{
 						"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef",
 						"0x0000000000000000000000002aacf811ac1a60081ea39f7783c0d26c500871a8",
@@ -30,11 +29,11 @@ func TestErc20_erc20GetTransfersFromLog(t *testing.T) {
 					Data: "0x0000000000000000000000000000000000000000000000000000000000000123",
 				},
 			},
-			want: []erc20Transfer{
+			want: []Erc20Transfer{
 				{
-					Contract: ethcommon.HexToAddress("0x76a45e8976499ab9ae223cc584019341d5a84e96"),
+					Contract: "0x76a45e8976499ab9ae223cc584019341d5a84e96",
-					From:     ethcommon.HexToAddress("0x2aacf811ac1a60081ea39f7783c0d26c500871a8"),
+					From:     "0x2aacf811ac1a60081ea39f7783c0d26c500871a8",
-					To:       ethcommon.HexToAddress("0xe9a5216ff992cfa01594d43501a56e12769eb9d2"),
+					To:       "0xe9a5216ff992cfa01594d43501a56e12769eb9d2",
 					Tokens:   *big.NewInt(0x123),
 				},
 			},
@@ -43,7 +42,7 @@ func TestErc20_erc20GetTransfersFromLog(t *testing.T) {
 			name: "2",
 			args: []*rpcLog{
 				&rpcLog{ // Transfer
-					Address: ethcommon.HexToAddress("0x0d0f936ee4c93e25944694d6c121de94d9760f11"),
+					Address: "0x0d0f936ee4c93e25944694d6c121de94d9760f11",
 					Topics: []string{
 						"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef",
 						"0x0000000000000000000000006f44cceb49b4a5812d54b6f494fc2febf25511ed",
@@ -52,7 +51,7 @@ func TestErc20_erc20GetTransfersFromLog(t *testing.T) {
 					Data: "0x0000000000000000000000000000000000000000000000006a8313d60b1f606b",
 				},
 				&rpcLog{ // Transfer
-					Address: ethcommon.HexToAddress("0xc778417e063141139fce010982780140aa0cd5ab"),
+					Address: "0xc778417e063141139fce010982780140aa0cd5ab",
 					Topics: []string{
 						"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef",
 						"0x0000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d",
@@ -61,7 +60,7 @@ func TestErc20_erc20GetTransfersFromLog(t *testing.T) {
 					Data: "0x000000000000000000000000000000000000000000000000000308fd0e798ac0",
 				},
 				&rpcLog{ // not Transfer
-					Address: ethcommon.HexToAddress("0x479cc461fecd078f766ecc58533d6f69580cf3ac"),
+					Address: "0x479cc461fecd078f766ecc58533d6f69580cf3ac",
 					Topics: []string{
 						"0x0d0b9391970d9a25552f37d436d2aae2925e2bfe1b2a923754bada030c498cb3",
 						"0x0000000000000000000000006f44cceb49b4a5812d54b6f494fc2febf25511ed",
@@ -71,7 +70,7 @@ func TestErc20_erc20GetTransfersFromLog(t *testing.T) {
 					Data: "0x0000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d0000000000000",
 				},
 				&rpcLog{ // not Transfer
-					Address: ethcommon.HexToAddress("0x0d0f936ee4c93e25944694d6c121de94d9760f11"),
+					Address: "0x0d0f936ee4c93e25944694d6c121de94d9760f11",
 					Topics: []string{
 						"0x0d0b9391970d9a25552f37d436d2aae2925e2bfe1b2a923754bada030c498cb3",
 						"0x0000000000000000000000007b62eb7fe80350dc7ec945c0b73242cb9877fb1b",
@@ -80,17 +79,17 @@ func TestErc20_erc20GetTransfersFromLog(t *testing.T) {
 					Data: "0x0000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d000000000000000000000000c778417e063141139fce010982780140aa0cd5ab0000000000000000000000000d0f936ee4c93e25944694d6c121de94d9760f1100000000000000000000000000000000000000000000000000031855667df7a80000000000000000000000000000000000000000000000006a8313d60b1f800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
 				},
 			},
-			want: []erc20Transfer{
+			want: []Erc20Transfer{
 				{
-					Contract: ethcommon.HexToAddress("0x0d0f936ee4c93e25944694d6c121de94d9760f11"),
+					Contract: "0x0d0f936ee4c93e25944694d6c121de94d9760f11",
-					From:     ethcommon.HexToAddress("0x6f44cceb49b4a5812d54b6f494fc2febf25511ed"),
+					From:     "0x6f44cceb49b4a5812d54b6f494fc2febf25511ed",
-					To:       ethcommon.HexToAddress("0x4bda106325c335df99eab7fe363cac8a0ba2a24d"),
+					To:       "0x4bda106325c335df99eab7fe363cac8a0ba2a24d",
 					Tokens:   *big.NewInt(0x6a8313d60b1f606b),
 				},
 				{
-					Contract: ethcommon.HexToAddress("0xc778417e063141139fce010982780140aa0cd5ab"),
+					Contract: "0xc778417e063141139fce010982780140aa0cd5ab",
-					From:     ethcommon.HexToAddress("0x4bda106325c335df99eab7fe363cac8a0ba2a24d"),
+					From:     "0x4bda106325c335df99eab7fe363cac8a0ba2a24d",
-					To:       ethcommon.HexToAddress("0x6f44cceb49b4a5812d54b6f494fc2febf25511ed"),
+					To:       "0x6f44cceb49b4a5812d54b6f494fc2febf25511ed",
 					Tokens:   *big.NewInt(0x308fd0e798ac0),
 				},
 			},
@@ -103,7 +102,8 @@ func TestErc20_erc20GetTransfersFromLog(t *testing.T) {
 				t.Errorf("erc20GetTransfersFromLog error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
-			if !reflect.DeepEqual(got, tt.want) {
+			// the addresses could have different case
+			if strings.ToLower(fmt.Sprint(got)) != strings.ToLower(fmt.Sprint(tt.want)) {
 				t.Errorf("erc20GetTransfersFromLog = %+v, want %+v", got, tt.want)
 			}
 		})

bchain/coins/eth/ethparser.go

@@ -7,21 +7,24 @@ import (
 	"math/big"
 	"strconv"
 
-	ethcommon "github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/golang/glog"
 	"github.com/golang/protobuf/proto"
 	"github.com/juju/errors"
 )
 
+// EthereumTypeAddressDescriptorLen - in case of EthereumType, the AddressDescriptor has fixed length
+const EthereumTypeAddressDescriptorLen = 20
+
 // EthereumParser handle
 type EthereumParser struct {
 	*bchain.BaseParser
 }
 
 // NewEthereumParser returns new EthereumParser instance
-func NewEthereumParser() *EthereumParser {
+func NewEthereumParser(b int) *EthereumParser {
 	return &EthereumParser{&bchain.BaseParser{
-		BlockAddressesToKeep: 0,
+		BlockAddressesToKeep: b,
 		AmountDecimalPoint:   18,
 	}}
 }
@@ -54,9 +57,9 @@ type rpcTransaction struct {
 }
 
 type rpcLog struct {
-	Address ethcommon.Address `json:"address"`
+	Address string   `json:"address"`
-	Topics  []string          `json:"topics"`
+	Topics  []string `json:"topics"`
-	Data    string            `json:"data"`
+	Data    string   `json:"data"`
 }
 
 type rpcLogWithTxHash struct {
@@ -171,7 +174,10 @@ func (p *EthereumParser) GetAddrDescFromAddress(address string) (bchain.AddressD
 	if has0xPrefix(address) {
 		address = address[2:]
 	}
-	if len(address) == 0 {
+	if len(address) != EthereumTypeAddressDescriptorLen*2 {
+		if len(address) != 0 {
+			glog.Warning("Ignoring address ", address)
+		}
 		return nil, bchain.ErrAddressMissing
 	}
 	if len(address)&1 == 1 {
@@ -278,6 +284,10 @@ func (p *EthereumParser) PackTx(tx *bchain.Tx, height uint32, blockTime int64) (
 		}
 		ptLogs := make([]*ProtoCompleteTransaction_ReceiptType_LogType, len(r.Receipt.Logs))
 		for i, l := range r.Receipt.Logs {
+			a, err := hexutil.Decode(l.Address)
+			if err != nil {
+				return nil, errors.Annotatef(err, "Address cannot be decoded %v", l)
+			}
 			d, err := hexutil.Decode(l.Data)
 			if err != nil {
 				return nil, errors.Annotatef(err, "Data cannot be decoded %v", l)
@@ -290,7 +300,7 @@ func (p *EthereumParser) PackTx(tx *bchain.Tx, height uint32, blockTime int64) (
 				}
 			}
 			ptLogs[i] = &ProtoCompleteTransaction_ReceiptType_LogType{
-				Address: l.Address.Bytes(),
+				Address: a,
 				Data:    d,
 				Topics:  t,
 			}
@@ -332,7 +342,7 @@ func (p *EthereumParser) UnpackTx(buf []byte) (*bchain.Tx, uint32, error) {
 				topics[j] = hexutil.Encode(t)
 			}
 			logs[i] = &rpcLog{
-				Address: ethcommon.BytesToAddress(l.Address),
+				Address: hexutil.Encode(l.Address),
 				Data:    hexutil.Encode(l.Data),
 				Topics:  topics,
 			}
@@ -388,19 +398,39 @@ func (p *EthereumParser) GetChainType() bchain.ChainType {
 
 // GetHeightFromTx returns ethereum specific data from bchain.Tx
 func GetHeightFromTx(tx *bchain.Tx) (uint32, error) {
-	// TODO -  temporary implementation - will use bchain.Tx.SpecificData field
+	var bn string
-	b, err := hex.DecodeString(tx.Hex)
+	csd, ok := tx.CoinSpecificData.(completeTransaction)
-	if err != nil {
+	if !ok {
-		return 0, err
+		b, err := hex.DecodeString(tx.Hex)
+		if err != nil {
+			return 0, err
+		}
+		var ct completeTransaction
+		err = json.Unmarshal(b, &ct)
+		if err != nil {
+			return 0, err
+		}
+		bn = ct.Tx.BlockNumber
+	} else {
+		bn = csd.Tx.BlockNumber
 	}
-	var ct completeTransaction
+	n, err := hexutil.DecodeUint64(bn)
-	var n uint64
-	err = json.Unmarshal(b, &ct)
 	if err != nil {
-		return 0, err
+		return 0, errors.Annotatef(err, "BlockNumber %v", bn)
-	}
-	if n, err = hexutil.DecodeUint64(ct.Tx.BlockNumber); err != nil {
-		return 0, errors.Annotatef(err, "BlockNumber %v", ct.Tx.BlockNumber)
 	}
 	return uint32(n), nil
 }
+
+// GetErc20FromTx returns Erc20 data from bchain.Tx
+func GetErc20FromTx(tx *bchain.Tx) ([]Erc20Transfer, error) {
+	var r []Erc20Transfer
+	var err error
+	csd, ok := tx.CoinSpecificData.(completeTransaction)
+	if ok {
+		r, err = erc20GetTransfersFromLog(csd.Receipt.Logs)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return r, nil
+}

bchain/coins/eth/ethparser_test.go

@@ -32,9 +32,10 @@ func TestEthParser_GetAddrDescFromAddress(t *testing.T) {
 			want: "47526228d673e9f079630d6cdaff5a2ed13e0e60",
 		},
 		{
-			name: "odd address",
+			name:    "address of wrong length",
-			args: args{address: "7526228d673e9f079630d6cdaff5a2ed13e0e60"},
+			args:    args{address: "7526228d673e9f079630d6cdaff5a2ed13e0e60"},
-			want: "07526228d673e9f079630d6cdaff5a2ed13e0e60",
+			want:    "",
+			wantErr: true,
 		},
 		{
 			name:    "ErrAddressMissing",
@@ -51,7 +52,7 @@ func TestEthParser_GetAddrDescFromAddress(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			p := NewEthereumParser()
+			p := NewEthereumParser(1)
 			got, err := p.GetAddrDescFromAddress(tt.args.address)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("EthParser.GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr)
@@ -146,7 +147,7 @@ func TestEthereumParser_PackTx(t *testing.T) {
 			want: testTxPacked2,
 		},
 	}
-	p := NewEthereumParser()
+	p := NewEthereumParser(1)
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			got, err := p.PackTx(tt.args.tx, tt.args.height, tt.args.blockTime)
@@ -187,7 +188,7 @@ func TestEthereumParser_UnpackTx(t *testing.T) {
 			want1: 2871048,
 		},
 	}
-	p := NewEthereumParser()
+	p := NewEthereumParser(1)
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			b, err := hex.DecodeString(tt.args.hex)

bchain/coins/eth/ethrpc.go

@@ -31,10 +31,11 @@ const (
 
 // Configuration represents json config file
 type Configuration struct {
-	CoinName     string `json:"coin_name"`
+	CoinName             string `json:"coin_name"`
-	CoinShortcut string `json:"coin_shortcut"`
+	CoinShortcut         string `json:"coin_shortcut"`
-	RPCURL       string `json:"rpc_url"`
+	RPCURL               string `json:"rpc_url"`
-	RPCTimeout   int    `json:"rpc_timeout"`
+	RPCTimeout           int    `json:"rpc_timeout"`
+	BlockAddressesToKeep int    `json:"block_addresses_to_keep"`
 }
 
 // EthereumRPC is an interface to JSON-RPC eth service.
@@ -65,6 +66,10 @@ func NewEthereumRPC(config json.RawMessage, pushHandler func(bchain.Notification
 	if err != nil {
 		return nil, errors.Annotatef(err, "Invalid configuration file")
 	}
+	// keep at least 100 mappings block->addresses to allow rollback
+	if c.BlockAddressesToKeep < 100 {
+		c.BlockAddressesToKeep = 100
+	}
 	rc, err := rpc.Dial(c.RPCURL)
 	if err != nil {
 		return nil, err
@@ -78,7 +83,7 @@ func NewEthereumRPC(config json.RawMessage, pushHandler func(bchain.Notification
 	}
 
 	// always create parser
-	s.Parser = NewEthereumParser()
+	s.Parser = NewEthereumParser(c.BlockAddressesToKeep)
 	s.timeout = time.Duration(c.RPCTimeout) * time.Second
 
 	// detect ethereum classic

db/rocksdb.go

@@ -59,13 +59,21 @@ const (
 	cfDefault = iota
 	cfHeight
 	cfAddresses
-	cfTxAddresses
-	cfAddressBalance
 	cfBlockTxs
 	cfTransactions
+	// BitcoinType
+	cfAddressBalance
+	cfTxAddresses
+	// EthereumType
+	cfAddressContracts = cfAddressBalance
 )
 
-var cfNames = []string{"default", "height", "addresses", "txAddresses", "addressBalance", "blockTxs", "transactions"}
+// common columns
+var cfNames = []string{"default", "height", "addresses", "blockTxs", "transactions"}
+
+// type specific columns
+var cfNamesBitcoinType = []string{"addressBalance", "txAddresses"}
+var cfNamesEthereumType = []string{"addressContracts"}
 
 func openDB(path string, c *gorocksdb.Cache, openFiles int) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) {
 	// opts with bloom filter
@@ -73,9 +81,14 @@ func openDB(path string, c *gorocksdb.Cache, openFiles int) (*gorocksdb.DB, []*g
 	// opts for addresses without bloom filter
 	// from documentation: if most of your queries are executed using iterators, you shouldn't set bloom filter
 	optsAddresses := createAndSetDBOptions(0, c, openFiles)
-	// default, height, addresses, txAddresses, addressBalance, blockTxids, transactions
+	// default, height, addresses, blockTxids, transactions
-	fcOptions := []*gorocksdb.Options{opts, opts, optsAddresses, opts, opts, opts, opts}
+	cfOptions := []*gorocksdb.Options{opts, opts, optsAddresses, opts, opts}
-	db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions)
+	// append type specific options
+	count := len(cfNames) - len(cfOptions)
+	for i := 0; i < count; i++ {
+		cfOptions = append(cfOptions, opts)
+	}
+	db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, cfOptions)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -86,6 +99,16 @@ func openDB(path string, c *gorocksdb.Cache, openFiles int) (*gorocksdb.DB, []*g
 // needs to be called to release it.
 func NewRocksDB(path string, cacheSize, maxOpenFiles int, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) {
 	glog.Infof("rocksdb: opening %s, required data version %v, cache size %v, max open files %v", path, dbVersion, cacheSize, maxOpenFiles)
+
+	chainType := parser.GetChainType()
+	if chainType == bchain.ChainBitcoinType {
+		cfNames = append(cfNames, cfNamesBitcoinType...)
+	} else if chainType == bchain.ChainEthereumType {
+		cfNames = append(cfNames, cfNamesEthereumType...)
+	} else {
+		return nil, errors.New("Unknown chain type")
+	}
+
 	c := gorocksdb.NewLRUCache(cacheSize)
 	db, cfh, err := openDB(path, c, maxOpenFiles)
 	if err != nil {
@@ -276,21 +299,18 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error {
 	if err := d.writeHeightFromBlock(wb, block, op); err != nil {
 		return err
 	}
+	addresses := make(map[string][]outpoint)
 	if chainType == bchain.ChainBitcoinType {
 		if op == opDelete {
 			// block does not contain mapping tx-> input address, which is necessary to recreate
 			// unspentTxs; therefore it is not possible to DisconnectBlocks this way
 			return errors.New("DisconnectBlock is not supported for BitcoinType chains")
 		}
-		addresses := make(map[string][]outpoint)
 		txAddressesMap := make(map[string]*TxAddresses)
 		balances := make(map[string]*AddrBalance)
 		if err := d.processAddressesBitcoinType(block, addresses, txAddressesMap, balances); err != nil {
 			return err
 		}
-		if err := d.storeAddresses(wb, block.Height, addresses); err != nil {
-			return err
-		}
 		if err := d.storeTxAddresses(wb, txAddressesMap); err != nil {
 			return err
 		}
@@ -301,12 +321,23 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error {
 			return err
 		}
 	} else if chainType == bchain.ChainEthereumType {
-		if err := d.writeAddressesEthereumType(wb, block, op); err != nil {
+		addressContracts := make(map[string]*AddrContracts)
+		blockTxs, err := d.processAddressesEthereumType(block, addresses, addressContracts)
+		if err != nil {
+			return err
+		}
+		if err := d.storeAddressContracts(wb, addressContracts); err != nil {
+			return err
+		}
+		if err := d.storeAndCleanupBlockTxsEthereumType(wb, block, blockTxs); err != nil {
 			return err
 		}
 	} else {
 		return errors.New("Unknown chain type")
 	}
+	if err := d.storeAddresses(wb, block.Height, addresses); err != nil {
+		return err
+	}
 
 	return d.db.Write(d.wo, wb)
 }
@@ -579,6 +610,26 @@ func (d *RocksDB) storeBalances(wb *gorocksdb.WriteBatch, abm map[string]*AddrBa
 	return nil
 }
 
+func (d *RocksDB) cleanupBlockTxs(wb *gorocksdb.WriteBatch, block *bchain.Block) error {
+	keep := d.chainParser.KeepBlockAddresses()
+	// cleanup old block address
+	if block.Height > uint32(keep) {
+		for rh := block.Height - uint32(keep); rh < block.Height; rh-- {
+			key := packUint(rh)
+			val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxs], key)
+			if err != nil {
+				return err
+			}
+			if val.Size() == 0 {
+				break
+			}
+			val.Free()
+			d.db.DeleteCF(d.wo, d.cfh[cfBlockTxs], key)
+		}
+	}
+	return nil
+}
+
 func (d *RocksDB) storeAndCleanupBlockTxs(wb *gorocksdb.WriteBatch, block *bchain.Block) error {
 	pl := d.chainParser.PackedTxidLen()
 	buf := make([]byte, 0, pl*len(block.Txs))
@@ -612,23 +663,7 @@ func (d *RocksDB) storeAndCleanupBlockTxs(wb *gorocksdb.WriteBatch, block *bchai
 	}
 	key := packUint(block.Height)
 	wb.PutCF(d.cfh[cfBlockTxs], key, buf)
-	keep := d.chainParser.KeepBlockAddresses()
+	return d.cleanupBlockTxs(wb, block)
-	// cleanup old block address
-	if block.Height > uint32(keep) {
-		for rh := block.Height - uint32(keep); rh < block.Height; rh-- {
-			key = packUint(rh)
-			val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxs], key)
-			if err != nil {
-				return err
-			}
-			if val.Size() == 0 {
-				break
-			}
-			val.Free()
-			d.db.DeleteCF(d.wo, d.cfh[cfBlockTxs], key)
-		}
-	}
-	return nil
 }
 
 func (d *RocksDB) getBlockTxs(height uint32) ([]blockTxs, error) {
@@ -844,74 +879,6 @@ func (d *RocksDB) unpackNOutpoints(buf []byte) ([]outpoint, int, error) {
 	return outpoints, p, nil
 }
 
-func (d *RocksDB) addAddrDescToRecords(op int, wb *gorocksdb.WriteBatch, records map[string][]outpoint, addrDesc bchain.AddressDescriptor, btxid []byte, vout int32, bh uint32) error {
-	if len(addrDesc) > 0 {
-		if len(addrDesc) > maxAddrDescLen {
-			glog.Infof("rocksdb: block %d, skipping addrDesc of length %d", bh, len(addrDesc))
-		} else {
-			strAddrDesc := string(addrDesc)
-			records[strAddrDesc] = append(records[strAddrDesc], outpoint{
-				btxID: btxid,
-				index: vout,
-			})
-			if op == opDelete {
-				// remove transactions from cache
-				d.internalDeleteTx(wb, btxid)
-			}
-		}
-	}
-	return nil
-}
-
-func (d *RocksDB) writeAddressesEthereumType(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error {
-	addresses := make(map[string][]outpoint)
-	for _, tx := range block.Txs {
-		btxID, err := d.chainParser.PackTxid(tx.Txid)
-		if err != nil {
-			return err
-		}
-		for _, output := range tx.Vout {
-			addrDesc, err := d.chainParser.GetAddrDescFromVout(&output)
-			if err != nil {
-				// do not log ErrAddressMissing, transactions can be without to address (for example eth contracts)
-				if err != bchain.ErrAddressMissing {
-					glog.Warningf("rocksdb: addrDesc: %v - height %d, tx %v, output %v", err, block.Height, tx.Txid, output)
-				}
-				continue
-			}
-			err = d.addAddrDescToRecords(op, wb, addresses, addrDesc, btxID, int32(output.N), block.Height)
-			if err != nil {
-				return err
-			}
-		}
-		// store inputs in format txid ^index
-		for _, input := range tx.Vin {
-			for i, a := range input.Addresses {
-				addrDesc, err := d.chainParser.GetAddrDescFromAddress(a)
-				if err != nil {
-					glog.Warningf("rocksdb: addrDesc: %v - %d %s", err, block.Height, addrDesc)
-					continue
-				}
-				err = d.addAddrDescToRecords(op, wb, addresses, addrDesc, btxID, int32(^i), block.Height)
-				if err != nil {
-					return err
-				}
-			}
-		}
-	}
-	for addrDesc, outpoints := range addresses {
-		key := packAddressKey(bchain.AddressDescriptor(addrDesc), block.Height)
-		switch op {
-		case opInsert:
-			val := d.packOutpoints(outpoints)
-			wb.PutCF(d.cfh[cfAddresses], key, val)
-		case opDelete:
-			wb.DeleteCF(d.cfh[cfAddresses], key)
-		}
-	}
-	return nil
-}
-
 // Block index
 
 // BlockInfo holds information about blocks kept in column height
@@ -1229,37 +1196,6 @@ func (d *RocksDB) DisconnectBlockRangeBitcoinType(lower uint32, higher uint32) e
 	return err
 }
 
-// DisconnectBlockRangeNonUTXO performs full range scan to remove a range of blocks
-// it is very slow operation
-func (d *RocksDB) DisconnectBlockRangeNonUTXO(lower uint32, higher uint32) error {
-	glog.Infof("db: disconnecting blocks %d-%d", lower, higher)
-	addrKeys, _, err := d.allAddressesScan(lower, higher)
-	if err != nil {
-		return err
-	}
-	glog.Infof("rocksdb: about to disconnect %d addresses ", len(addrKeys))
-	wb := gorocksdb.NewWriteBatch()
-	defer wb.Destroy()
-	for _, addrKey := range addrKeys {
-		if glog.V(2) {
-			glog.Info("address ", hex.EncodeToString(addrKey))
-		}
-		// delete address:height from the index
-		wb.DeleteCF(d.cfh[cfAddresses], addrKey)
-	}
-	for height := lower; height <= higher; height++ {
-		if glog.V(2) {
-			glog.Info("height ", height)
-		}
-		wb.DeleteCF(d.cfh[cfHeight], packUint(height))
-	}
-	err = d.db.Write(d.wo, wb)
-	if err == nil {
-		glog.Infof("rocksdb: blocks %d-%d disconnected", lower, higher)
-	}
-	return err
-}
-
 func dirSize(path string) (int64, error) {
 	var size int64
 	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {

db/rocksdb_ethereumtype.go

@@ -0,0 +1,272 @@
+package db
+
+import (
+	"blockbook/bchain"
+	"blockbook/bchain/coins/eth"
+	"bytes"
+	"encoding/hex"
+
+	"github.com/bsm/go-vlq"
+	"github.com/golang/glog"
+	"github.com/juju/errors"
+	"github.com/tecbot/gorocksdb"
+)
+
+// AddrContract is Contract address with number of transactions done by given address
+type AddrContract struct {
+	Contract bchain.AddressDescriptor
+	Txs      uint
+}
+
+// AddrContracts is array of contracts with
+type AddrContracts struct {
+	EthTxs    uint
+	Contracts []AddrContract
+}
+
+func (d *RocksDB) storeAddressContracts(wb *gorocksdb.WriteBatch, acm map[string]*AddrContracts) error {
+	buf := make([]byte, 64)
+	varBuf := make([]byte, vlq.MaxLen64)
+	for addrDesc, acs := range acm {
+		// address with 0 contracts is removed from db - happens on disconnect
+		if acs == nil || (acs.EthTxs == 0 && len(acs.Contracts) == 0) {
+			wb.DeleteCF(d.cfh[cfAddressContracts], bchain.AddressDescriptor(addrDesc))
+		} else {
+			buf = buf[:0]
+			l := packVaruint(acs.EthTxs, varBuf)
+			buf = append(buf, varBuf[:l]...)
+			for _, ac := range acs.Contracts {
+				buf = append(buf, ac.Contract...)
+				l = packVaruint(ac.Txs, varBuf)
+				buf = append(buf, varBuf[:l]...)
+			}
+			wb.PutCF(d.cfh[cfAddressContracts], bchain.AddressDescriptor(addrDesc), buf)
+		}
+	}
+	return nil
+}
+
+// GetAddrDescContracts returns AddrContracts for given addrDesc
+func (d *RocksDB) GetAddrDescContracts(addrDesc bchain.AddressDescriptor) (*AddrContracts, error) {
+	val, err := d.db.GetCF(d.ro, d.cfh[cfAddressContracts], addrDesc)
+	if err != nil {
+		return nil, err
+	}
+	defer val.Free()
+	buf := val.Data()
+	if len(buf) == 0 {
+		return nil, nil
+	}
+	c := make([]AddrContract, 0, 4)
+	et, l := unpackVaruint(buf)
+	buf = buf[l:]
+	for len(buf) > 0 {
+		if len(buf) < eth.EthereumTypeAddressDescriptorLen {
+			return nil, errors.New("Invalid data stored in cfAddressContracts for AddrDesc " + addrDesc.String())
+		}
+		txs, l := unpackVaruint(buf[eth.EthereumTypeAddressDescriptorLen:])
+		contract := make(bchain.AddressDescriptor, eth.EthereumTypeAddressDescriptorLen)
+		copy(contract, buf[:eth.EthereumTypeAddressDescriptorLen])
+		c = append(c, AddrContract{
+			Contract: contract,
+			Txs:      txs,
+		})
+		buf = buf[eth.EthereumTypeAddressDescriptorLen+l:]
+	}
+	return &AddrContracts{
+		EthTxs:    et,
+		Contracts: c}, nil
+}
+
+func (d *RocksDB) addToAddressesAndContractsEthereumType(addrDesc bchain.AddressDescriptor, btxID []byte, index int32, contract bchain.AddressDescriptor, addresses map[string][]outpoint, addressContracts map[string]*AddrContracts) error {
+	var err error
+	strAddrDesc := string(addrDesc)
+	ac, e := addressContracts[strAddrDesc]
+	if !e {
+		ac, err = d.GetAddrDescContracts(addrDesc)
+		if err != nil {
+			return err
+		}
+		if ac == nil {
+			ac = &AddrContracts{}
+		}
+		addressContracts[strAddrDesc] = ac
+		d.cbs.balancesMiss++
+	} else {
+		d.cbs.balancesHit++
+	}
+	if contract == nil {
+		ac.EthTxs++
+	} else {
+		// locate the contract and set i to the index in the array of contracts
+		var i int
+		var found bool
+		for i = range ac.Contracts {
+			if bytes.Equal(contract, ac.Contracts[i].Contract) {
+				found = true
+				break
+			}
+		}
+		if !found {
+			i = len(ac.Contracts)
+			ac.Contracts = append(ac.Contracts, AddrContract{Contract: contract})
+		}
+		// index 0 is for ETH transfers, contract indexes start with 1
+		if index < 0 {
+			index = ^int32(i + 1)
+		} else {
+			index = int32(i + 1)
+		}
+		ac.Contracts[i].Txs++
+	}
+	addresses[strAddrDesc] = append(addresses[strAddrDesc], outpoint{
+		btxID: btxID,
+		index: index,
+	})
+	return nil
+}
+
+type ethBlockTxContract struct {
+	addr, contract bchain.AddressDescriptor
+}
+
+type ethBlockTx struct {
+	btxID     []byte
+	from, to  bchain.AddressDescriptor
+	contracts []ethBlockTxContract
+}
+
+func (d *RocksDB) processAddressesEthereumType(block *bchain.Block, addresses map[string][]outpoint, addressContracts map[string]*AddrContracts) ([]ethBlockTx, error) {
+	blockTxs := make([]ethBlockTx, len(block.Txs))
+	for txi, tx := range block.Txs {
+		btxID, err := d.chainParser.PackTxid(tx.Txid)
+		if err != nil {
+			return nil, err
+		}
+		blockTx := &blockTxs[txi]
+		blockTx.btxID = btxID
+		// there is only one output address in EthereumType transaction, store it in format txid 0
+		if len(tx.Vout) == 1 && len(tx.Vout[0].ScriptPubKey.Addresses) == 1 {
+			addrDesc, err := d.chainParser.GetAddrDescFromAddress(tx.Vout[0].ScriptPubKey.Addresses[0])
+			if err != nil {
+				// do not log ErrAddressMissing, transactions can be without to address (for example eth contracts)
+				if err != bchain.ErrAddressMissing {
+					glog.Warningf("rocksdb: addrDesc: %v - height %d, tx %v, output", err, block.Height, tx.Txid)
+				}
+				continue
+			}
+			if err = d.addToAddressesAndContractsEthereumType(addrDesc, btxID, 0, nil, addresses, addressContracts); err != nil {
+				return nil, err
+			}
+			blockTx.to = addrDesc
+		}
+		// there is only one input address in EthereumType transaction, store it in format txid ^0
+		if len(tx.Vin) == 1 && len(tx.Vin[0].Addresses) == 1 {
+			addrDesc, err := d.chainParser.GetAddrDescFromAddress(tx.Vin[0].Addresses[0])
+			if err != nil {
+				if err != bchain.ErrAddressMissing {
+					glog.Warningf("rocksdb: addrDesc: %v - height %d, tx %v, input", err, block.Height, tx.Txid)
+				}
+				continue
+			}
+			if err = d.addToAddressesAndContractsEthereumType(addrDesc, btxID, ^int32(0), nil, addresses, addressContracts); err != nil {
+				return nil, err
+			}
+			blockTx.from = addrDesc
+		}
+		// store erc20 transfers
+		erc20, err := eth.GetErc20FromTx(&tx)
+		if err != nil {
+			glog.Warningf("rocksdb: GetErc20FromTx %v - height %d, tx %v", err, block.Height, tx.Txid)
+		}
+		blockTx.contracts = make([]ethBlockTxContract, len(erc20)*2)
+		for i, t := range erc20 {
+			var contract, from, to bchain.AddressDescriptor
+			contract, err = d.chainParser.GetAddrDescFromAddress(t.Contract)
+			if err == nil {
+				from, err = d.chainParser.GetAddrDescFromAddress(t.From)
+				if err == nil {
+					to, err = d.chainParser.GetAddrDescFromAddress(t.To)
+				}
+			}
+			if err != nil {
+				glog.Warningf("rocksdb: GetErc20FromTx %v - height %d, tx %v, transfer %v", err, block.Height, tx.Txid, t)
+				continue
+			}
+			if err = d.addToAddressesAndContractsEthereumType(from, btxID, ^int32(i), contract, addresses, addressContracts); err != nil {
+				return nil, err
+			}
+			bc := &blockTx.contracts[i*2]
+			bc.addr = from
+			bc.contract = contract
+			if err = d.addToAddressesAndContractsEthereumType(to, btxID, int32(i), contract, addresses, addressContracts); err != nil {
+				return nil, err
+			}
+			bc = &blockTx.contracts[i*2+1]
+			bc.addr = to
+			bc.contract = contract
+		}
+	}
+	return blockTxs, nil
+}
+
+func (d *RocksDB) storeAndCleanupBlockTxsEthereumType(wb *gorocksdb.WriteBatch, block *bchain.Block, blockTxs []ethBlockTx) error {
+	pl := d.chainParser.PackedTxidLen()
+	buf := make([]byte, 0, (pl+2*eth.EthereumTypeAddressDescriptorLen)*len(blockTxs))
+	varBuf := make([]byte, vlq.MaxLen64)
+	zeroAddress := make([]byte, eth.EthereumTypeAddressDescriptorLen)
+	appendAddress := func(a bchain.AddressDescriptor) {
+		if len(a) != eth.EthereumTypeAddressDescriptorLen {
+			buf = append(buf, zeroAddress...)
+		} else {
+			buf = append(buf, a...)
+		}
+	}
+	for i := range blockTxs {
+		blockTx := &blockTxs[i]
+		buf = append(buf, blockTx.btxID...)
+		appendAddress(blockTx.from)
+		appendAddress(blockTx.to)
+		l := packVaruint(uint(len(blockTx.contracts)), varBuf)
+		buf = append(buf, varBuf[:l]...)
+		for j := range blockTx.contracts {
+			c := &blockTx.contracts[j]
+			appendAddress(c.addr)
+			appendAddress(c.contract)
+		}
+	}
+	key := packUint(block.Height)
+	wb.PutCF(d.cfh[cfBlockTxs], key, buf)
+	return d.cleanupBlockTxs(wb, block)
+}
+
+// DisconnectBlockRangeNonUTXO performs full range scan to remove a range of blocks
+// it is very slow operation
+func (d *RocksDB) DisconnectBlockRangeNonUTXO(lower uint32, higher uint32) error {
+	glog.Infof("db: disconnecting blocks %d-%d", lower, higher)
+	addrKeys, _, err := d.allAddressesScan(lower, higher)
+	if err != nil {
+		return err
+	}
+	glog.Infof("rocksdb: about to disconnect %d addresses ", len(addrKeys))
+	wb := gorocksdb.NewWriteBatch()
+	defer wb.Destroy()
+	for _, addrKey := range addrKeys {
+		if glog.V(2) {
+			glog.Info("address ", hex.EncodeToString(addrKey))
+		}
+		// delete address:height from the index
+		wb.DeleteCF(d.cfh[cfAddresses], addrKey)
+	}
+	for height := lower; height <= higher; height++ {
+		if glog.V(2) {
+			glog.Info("height ", height)
+		}
+		wb.DeleteCF(d.cfh[cfHeight], packUint(height))
+	}
+	err = d.db.Write(d.wo, wb)
+	if err == nil {
+		glog.Infof("rocksdb: blocks %d-%d disconnected", lower, higher)
+	}
+	return err
+}

db/rocksdb_ethereumtype_test.go

@@ -0,0 +1,183 @@
+// build unittest
+
+package db
+
+import (
+	"blockbook/bchain/coins/eth"
+	"blockbook/tests/dbtestdata"
+	"testing"
+)
+
+type testEthereumParser struct {
+	*eth.EthereumParser
+}
+
+func ethereumTestnetParser() *eth.EthereumParser {
+	return eth.NewEthereumParser(1)
+}
+
+func verifyAfterEthereumTypeBlock1(t *testing.T, d *RocksDB, afterDisconnect bool) {
+	if err := checkColumn(d, cfHeight, []keyPair{
+		keyPair{
+			"0041eee8",
+			"c7b98df95acfd11c51ba25611a39e004fe56c8fdfc1582af99354fcd09c17b11" + uintToHex(1534858022) + varuintToHex(2) + varuintToHex(31839),
+			nil,
+		},
+	}); err != nil {
+		{
+			t.Fatal(err)
+		}
+	}
+	// the vout is encoded as signed varint, i.e. value * 2 for positive values, abs(value)*2 + 1 for negative values
+	if err := checkColumn(d, cfAddresses, []keyPair{
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "01", nil},
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "00" + dbtestdata.EthTxidB1T2 + "02", nil},
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "01" + dbtestdata.EthTxidB1T2 + "03", nil},
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "00", nil},
+	}); err != nil {
+		{
+			t.Fatal(err)
+		}
+	}
+
+	if err := checkColumn(d, cfAddressContracts, []keyPair{
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser), "01", nil},
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser), "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01", nil},
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser), "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01", nil},
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser), "01", nil},
+	}); err != nil {
+		{
+			t.Fatal(err)
+		}
+	}
+
+	var blockTxsKp []keyPair
+	if afterDisconnect {
+		blockTxsKp = []keyPair{}
+	} else {
+		blockTxsKp = []keyPair{
+			keyPair{
+				"0041eee8",
+				dbtestdata.EthTxidB1T1 +
+					dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "00" +
+					dbtestdata.EthTxidB1T2 +
+					dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) +
+					"02" +
+					dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) +
+					dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser),
+				nil,
+			},
+		}
+	}
+	if err := checkColumn(d, cfBlockTxs, blockTxsKp); err != nil {
+		{
+			t.Fatal(err)
+		}
+	}
+}
+
+func verifyAfterEthereumTypeBlock2(t *testing.T, d *RocksDB) {
+	if err := checkColumn(d, cfHeight, []keyPair{
+		keyPair{
+			"0041eee8",
+			"c7b98df95acfd11c51ba25611a39e004fe56c8fdfc1582af99354fcd09c17b11" + uintToHex(1534858022) + varuintToHex(2) + varuintToHex(31839),
+			nil,
+		},
+		keyPair{
+			"0041eee9",
+			"00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6" + uintToHex(1534859988) + varuintToHex(2) + varuintToHex(2345678),
+			nil,
+		},
+	}); err != nil {
+		{
+			t.Fatal(err)
+		}
+	}
+	// the vout is encoded as signed varint, i.e. value * 2 for positive values, abs(value)*2 + 1 for negative values
+	if err := checkColumn(d, cfAddresses, []keyPair{
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "01", nil},
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "00" + dbtestdata.EthTxidB1T2 + "02", nil},
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "01" + dbtestdata.EthTxidB1T2 + "03", nil},
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "00", nil},
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T1 + "01" + dbtestdata.EthTxidB2T2 + "05" + dbtestdata.EthTxidB2T2 + "02", nil},
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr9f, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T1 + "00", nil},
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T2 + "01" + dbtestdata.EthTxidB2T2 + "02" + dbtestdata.EthTxidB2T2 + "05" + dbtestdata.EthTxidB2T2 + "04" + dbtestdata.EthTxidB2T2 + "03", nil},
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr7b, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T2 + "03" + dbtestdata.EthTxidB2T2 + "04", nil},
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract47, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T2 + "00", nil},
+	}); err != nil {
+		{
+			t.Fatal(err)
+		}
+	}
+
+	if err := checkColumn(d, cfAddressContracts, []keyPair{
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser), "01", nil},
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser), "02" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "02" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + "01", nil},
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser), "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01", nil},
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser), "01", nil},
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr9f, d.chainParser), "01", nil},
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser), "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + "02" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "02", nil},
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr7b, d.chainParser), "00" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + "01", nil},
+		keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract47, d.chainParser), "01", nil},
+	}); err != nil {
+		{
+			t.Fatal(err)
+		}
+	}
+
+	if err := checkColumn(d, cfBlockTxs, []keyPair{
+		keyPair{
+			"0041eee9",
+			dbtestdata.EthTxidB2T1 +
+				dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr9f, d.chainParser) + "00" +
+				dbtestdata.EthTxidB2T2 +
+				dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract47, d.chainParser) +
+				"08" +
+				dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) +
+				dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) +
+				dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) +
+				dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) +
+				dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr7b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) +
+				dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) +
+				dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) +
+				dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr7b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser),
+			nil,
+		},
+	}); err != nil {
+		{
+			t.Fatal(err)
+		}
+	}
+}
+
+// TestRocksDB_Index_EthereumType is an integration test probing the whole indexing functionality for EthereumType chains
+// It does the following:
+// 1) Connect two blocks (inputs from 2nd block are spending some outputs from the 1st block)
+// 2) GetTransactions for various addresses / low-high ranges
+// 3) GetBestBlock, GetBlockHash
+// 4) Test tx caching functionality
+// 5) Disconnect block 2 - expect error
+// 6) Disconnect the block 2 using BlockTxs column
+// 7) Reconnect block 2 and check
+// After each step, the content of DB is examined and any difference against expected state is regarded as failure
+func TestRocksDB_Index_EthereumType(t *testing.T) {
+	d := setupRocksDB(t, &testEthereumParser{
+		EthereumParser: ethereumTestnetParser(),
+	})
+	defer closeAndDestroyRocksDB(t, d)
+
+	// connect 1st block - will log warnings about missing UTXO transactions in txAddresses column
+	block1 := dbtestdata.GetTestEthereumTypeBlock1(d.chainParser)
+	if err := d.ConnectBlock(block1); err != nil {
+		t.Fatal(err)
+	}
+	verifyAfterEthereumTypeBlock1(t, d, false)
+
+	// connect 2nd block - use some outputs from the 1st block as the inputs and 1 input uses tx from the same block
+	block2 := dbtestdata.GetTestEthereumTypeBlock2(d.chainParser)
+	if err := d.ConnectBlock(block2); err != nil {
+		t.Fatal(err)
+	}
+	verifyAfterEthereumTypeBlock2(t, d)
+
+}

db/rocksdb_test.go

@@ -33,6 +33,10 @@ func TestMain(m *testing.M) {
 	os.Exit(c)
 }
 
+type testBitcoinParser struct {
+	*btc.BitcoinParser
+}
+
 func bitcoinTestnetParser() *btc.BitcoinParser {
 	return btc.NewBitcoinParser(
 		btc.GetChainParams("test"),
@@ -48,7 +52,7 @@ func setupRocksDB(t *testing.T, p bchain.BlockChainParser) *RocksDB {
 	if err != nil {
 		t.Fatal(err)
 	}
-	is, err := d.LoadInternalState("btc-testnet")
+	is, err := d.LoadInternalState("coin-unittest")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -151,7 +155,7 @@ func checkColumn(d *RocksDB, col int, kp []keyPair) error {
 	return nil
 }
 
-func verifyAfterBitcoinTypeOBlock1(t *testing.T, d *RocksDB, afterDisconnect bool) {
+func verifyAfterBitcoinTypeBlock1(t *testing.T, d *RocksDB, afterDisconnect bool) {
 	if err := checkColumn(d, cfHeight, []keyPair{
 		keyPair{
 			"000370d5",
@@ -388,10 +392,6 @@ func verifyGetTransactions(t *testing.T, d *RocksDB, addr string, low, high uint
 	}
 }
 
-type testBitcoinParser struct {
-	*btc.BitcoinParser
-}
-
 // override PackTx and UnpackTx to default BaseParser functionality
 // BitcoinParser uses tx hex which is not available for the test transactions
 func (p *testBitcoinParser) PackTx(tx *bchain.Tx, height uint32, blockTime int64) ([]byte, error) {
@@ -444,7 +444,7 @@ func TestRocksDB_Index_BitcoinType(t *testing.T) {
 	if err := d.ConnectBlock(block1); err != nil {
 		t.Fatal(err)
 	}
-	verifyAfterBitcoinTypeOBlock1(t, d, false)
+	verifyAfterBitcoinTypeBlock1(t, d, false)
 
 	// connect 2nd block - use some outputs from the 1st block as the inputs and 1 input uses tx from the same block
 	block2 := dbtestdata.GetTestBitcoinTypeBlock2(d.chainParser)
@@ -552,7 +552,7 @@ func TestRocksDB_Index_BitcoinType(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	verifyAfterBitcoinTypeOBlock1(t, d, true)
+	verifyAfterBitcoinTypeBlock1(t, d, true)
 	if err := checkColumn(d, cfTransactions, []keyPair{}); err != nil {
 		{
 			t.Fatal(err)