From 6d3e171b710bbdb2c367471ede7ff32467144f1f Mon Sep 17 00:00:00 2001
From: Martin Boehm <martin.boehm@1mbsoftware.net>
Date: Tue, 13 Nov 2018 10:31:27 +0100
Subject: [PATCH] Handle coin specific tx data more efficiently

---
 api/types.go                       | 37 +++++++++++-----------
 api/worker.go                      | 49 ++++++++++++++++++------------
 bchain/coins/blockchain.go         |  4 +--
 bchain/coins/btc/bitcoinrpc.go     | 13 ++++++--
 bchain/coins/eth/ethparser.go      | 31 ++++++++++---------
 bchain/coins/eth/ethparser_test.go |  2 ++
 bchain/coins/eth/ethrpc.go         | 29 ++++++++++--------
 bchain/types.go                    |  9 +++---
 server/public.go                   | 15 +++------
 server/socketio.go                 |  4 +--
 static/templates/tx.html           |  4 +--
 tests/dbtestdata/fakechain.go      |  4 +--
 12 files changed, 111 insertions(+), 90 deletions(-)

diff --git a/api/types.go b/api/types.go
index 8a535f24..9925bc7d 100644
--- a/api/types.go
+++ b/api/types.go
@@ -70,24 +70,25 @@ type Vout struct {
 
 // Tx holds information about a transaction
 type Tx struct {
-	Txid          string  `json:"txid"`
-	Version       int32   `json:"version,omitempty"`
-	Locktime      uint32  `json:"locktime,omitempty"`
-	Vin           []Vin   `json:"vin"`
-	Vout          []Vout  `json:"vout"`
-	Blockhash     string  `json:"blockhash,omitempty"`
-	Blockheight   int     `json:"blockheight"`
-	Confirmations uint32  `json:"confirmations"`
-	Time          int64   `json:"time,omitempty"`
-	Blocktime     int64   `json:"blocktime"`
-	ValueOut      string  `json:"valueOut"`
-	ValueOutSat   big.Int `json:"-"`
-	Size          int     `json:"size,omitempty"`
-	ValueIn       string  `json:"valueIn"`
-	ValueInSat    big.Int `json:"-"`
-	Fees          string  `json:"fees"`
-	FeesSat       big.Int `json:"-"`
-	Hex           string  `json:"hex"`
+	Txid             string      `json:"txid"`
+	Version          int32       `json:"version,omitempty"`
+	Locktime         uint32      `json:"locktime,omitempty"`
+	Vin              []Vin       `json:"vin"`
+	Vout             []Vout      `json:"vout"`
+	Blockhash        string      `json:"blockhash,omitempty"`
+	Blockheight      int         `json:"blockheight"`
+	Confirmations    uint32      `json:"confirmations"`
+	Time             int64       `json:"time,omitempty"`
+	Blocktime        int64       `json:"blocktime"`
+	ValueOut         string      `json:"valueOut"`
+	ValueOutSat      big.Int     `json:"-"`
+	Size             int         `json:"size,omitempty"`
+	ValueIn          string      `json:"valueIn"`
+	ValueInSat       big.Int     `json:"-"`
+	Fees             string      `json:"fees"`
+	FeesSat          big.Int     `json:"-"`
+	Hex              string      `json:"hex"`
+	CoinSpecificData interface{} `json:"-"`
 }
 
 // Paging contains information about paging for address, blocks and block
diff --git a/api/worker.go b/api/worker.go
index e30f3bdd..f1aa6538 100644
--- a/api/worker.go
+++ b/api/worker.go
@@ -5,6 +5,7 @@ import (
 	"blockbook/common"
 	"blockbook/db"
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"math/big"
 	"strconv"
@@ -82,7 +83,7 @@ func (w *Worker) setSpendingTxToVout(vout *Vout, txid string, height uint32) err
 // GetSpendingTxid returns transaction id of transaction that spent given output
 func (w *Worker) GetSpendingTxid(txid string, n int) (string, error) {
 	start := time.Now()
-	tx, err := w.GetTransaction(txid, false)
+	tx, err := w.GetTransaction(txid, false, false)
 	if err != nil {
 		return "", err
 	}
@@ -98,7 +99,7 @@ func (w *Worker) GetSpendingTxid(txid string, n int) (string, error) {
 }
 
 // GetTransaction reads transaction data from txid
-func (w *Worker) GetTransaction(txid string, spendingTxs bool) (*Tx, error) {
+func (w *Worker) GetTransaction(txid string, spendingTxs bool, specificData bool) (*Tx, error) {
 	start := time.Now()
 	bchainTx, height, err := w.txCache.GetTransaction(txid)
 	if err != nil {
@@ -210,24 +211,32 @@ func (w *Worker) GetTransaction(txid string, spendingTxs bool) (*Tx, error) {
 	}
 	// for now do not return size, we would have to compute vsize of segwit transactions
 	// size:=len(bchainTx.Hex) / 2
+	var sd json.RawMessage
+	if specificData {
+		sd, err = w.chain.GetTransactionSpecific(bchainTx)
+		if err != nil {
+			return nil, err
+		}
+	}
 	r := &Tx{
-		Blockhash:     blockhash,
-		Blockheight:   int(height),
-		Blocktime:     bchainTx.Blocktime,
-		Confirmations: bchainTx.Confirmations,
-		Fees:          w.chainParser.AmountToDecimalString(&feesSat),
-		FeesSat:       feesSat,
-		Locktime:      bchainTx.LockTime,
-		Time:          bchainTx.Time,
-		Txid:          bchainTx.Txid,
-		ValueIn:       w.chainParser.AmountToDecimalString(&valInSat),
-		ValueInSat:    valInSat,
-		ValueOut:      w.chainParser.AmountToDecimalString(&valOutSat),
-		ValueOutSat:   valOutSat,
-		Version:       bchainTx.Version,
-		Hex:           bchainTx.Hex,
-		Vin:           vins,
-		Vout:          vouts,
+		Blockhash:        blockhash,
+		Blockheight:      int(height),
+		Blocktime:        bchainTx.Blocktime,
+		Confirmations:    bchainTx.Confirmations,
+		Fees:             w.chainParser.AmountToDecimalString(&feesSat),
+		FeesSat:          feesSat,
+		Locktime:         bchainTx.LockTime,
+		Time:             bchainTx.Time,
+		Txid:             bchainTx.Txid,
+		ValueIn:          w.chainParser.AmountToDecimalString(&valInSat),
+		ValueInSat:       valInSat,
+		ValueOut:         w.chainParser.AmountToDecimalString(&valOutSat),
+		ValueOutSat:      valOutSat,
+		Version:          bchainTx.Version,
+		Hex:              bchainTx.Hex,
+		Vin:              vins,
+		Vout:             vouts,
+		CoinSpecificData: sd,
 	}
 	if spendingTxs {
 		glog.Info("GetTransaction ", txid, " finished in ", time.Since(start))
@@ -426,7 +435,7 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids b
 	// load mempool transactions
 	var uBalSat big.Int
 	for _, tx := range txm {
-		tx, err := w.GetTransaction(tx, false)
+		tx, err := w.GetTransaction(tx, false, false)
 		// mempool transaction may fail
 		if err != nil {
 			glog.Error("GetTransaction in mempool ", tx, ": ", err)
diff --git a/bchain/coins/blockchain.go b/bchain/coins/blockchain.go
index 2e02db45..cd6f58eb 100644
--- a/bchain/coins/blockchain.go
+++ b/bchain/coins/blockchain.go
@@ -187,9 +187,9 @@ func (c *blockChainWithMetrics) GetTransaction(txid string) (v *bchain.Tx, err e
 	return c.b.GetTransaction(txid)
 }
 
-func (c *blockChainWithMetrics) GetTransactionSpecific(txid string) (v json.RawMessage, err error) {
+func (c *blockChainWithMetrics) GetTransactionSpecific(tx *bchain.Tx) (v json.RawMessage, err error) {
 	defer func(s time.Time) { c.observeRPCLatency("GetTransactionSpecific", s, err) }(time.Now())
-	return c.b.GetTransactionSpecific(txid)
+	return c.b.GetTransactionSpecific(tx)
 }
 
 func (c *blockChainWithMetrics) GetTransactionForMempool(txid string) (v *bchain.Tx, err error) {
diff --git a/bchain/coins/btc/bitcoinrpc.go b/bchain/coins/btc/bitcoinrpc.go
index 8b69e8a2..f17d4db1 100644
--- a/bchain/coins/btc/bitcoinrpc.go
+++ b/bchain/coins/btc/bitcoinrpc.go
@@ -670,11 +670,12 @@ func (b *BitcoinRPC) GetTransactionForMempool(txid string) (*bchain.Tx, error) {
 
 // GetTransaction returns a transaction by the transaction ID
 func (b *BitcoinRPC) GetTransaction(txid string) (*bchain.Tx, error) {
-	r, err := b.GetTransactionSpecific(txid)
+	r, err := b.getRawTransaction(txid)
 	if err != nil {
 		return nil, err
 	}
 	tx, err := b.Parser.ParseTxFromJson(r)
+	tx.CoinSpecificData = r
 	if err != nil {
 		return nil, errors.Annotatef(err, "txid %v", txid)
 	}
@@ -682,7 +683,15 @@ func (b *BitcoinRPC) GetTransaction(txid string) (*bchain.Tx, error) {
 }
 
 // GetTransactionSpecific returns json as returned by backend, with all coin specific data
-func (b *BitcoinRPC) GetTransactionSpecific(txid string) (json.RawMessage, error) {
+func (b *BitcoinRPC) GetTransactionSpecific(tx *bchain.Tx) (json.RawMessage, error) {
+	if csd, ok := tx.CoinSpecificData.(json.RawMessage); ok {
+		return csd, nil
+	}
+	return b.getRawTransaction(tx.Txid)
+}
+
+// getRawTransaction returns json as returned by backend, with all coin specific data
+func (b *BitcoinRPC) getRawTransaction(txid string) (json.RawMessage, error) {
 	glog.V(1).Info("rpc: getrawtransaction ", txid)
 
 	res := ResGetRawTransaction{}
diff --git a/bchain/coins/eth/ethparser.go b/bchain/coins/eth/ethparser.go
index 4b92c676..48cce14b 100644
--- a/bchain/coins/eth/ethparser.go
+++ b/bchain/coins/eth/ethparser.go
@@ -79,7 +79,7 @@ func ethNumber(n string) (int64, error) {
 	return 0, errors.Errorf("Not a number: '%v'", n)
 }
 
-func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, receipt *rpcReceipt, blocktime int64, confirmations uint32) (*bchain.Tx, error) {
+func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, receipt *rpcReceipt, blocktime int64, confirmations uint32, marshallHex bool) (*bchain.Tx, error) {
 	txid := ethHashToHash(tx.Hash)
 	var (
 		fa, ta []string
@@ -91,22 +91,24 @@ func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, receipt *rpcReceipt, bloc
 	if len(tx.To) > 2 {
 		ta = []string{tx.To}
 	}
-
-	// completeTransaction without BlockHash is marshalled and hex encoded to bchain.Tx.Hex
-	bh := tx.BlockHash
-	tx.BlockHash = nil
 	ct := completeTransaction{
 		Tx:      tx,
 		Receipt: receipt,
 	}
-	b, err := json.Marshal(ct)
-	if err != nil {
-		return nil, err
-	}
-	tx.BlockHash = bh
-	h := hex.EncodeToString(b)
-	if receipt != nil {
-		glog.Info(tx.Hash.Hex(), ": ", h)
+	var h string
+	if marshallHex {
+		// completeTransaction without BlockHash is marshalled and hex encoded to bchain.Tx.Hex
+		bh := tx.BlockHash
+		tx.BlockHash = nil
+		b, err := json.Marshal(ct)
+		if err != nil {
+			return nil, err
+		}
+		tx.BlockHash = bh
+		h = hex.EncodeToString(b)
+		if receipt != nil {
+			glog.Info(tx.Hash.Hex(), ": ", h)
+		}
 	}
 	vs, err := hexutil.DecodeBig(tx.Value)
 	if err != nil {
@@ -139,6 +141,7 @@ func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, receipt *rpcReceipt, bloc
 				},
 			},
 		},
+		CoinSpecificData: ct,
 	}, nil
 }
 
@@ -330,7 +333,7 @@ func (p *EthereumParser) UnpackTx(buf []byte) (*bchain.Tx, uint32, error) {
 			Logs:    logs,
 		}
 	}
-	tx, err := p.ethTxToTx(&rt, rr, int64(pt.BlockTime), 0)
+	tx, err := p.ethTxToTx(&rt, rr, int64(pt.BlockTime), 0, true)
 	if err != nil {
 		return nil, 0, err
 	}
diff --git a/bchain/coins/eth/ethparser_test.go b/bchain/coins/eth/ethparser_test.go
index fb082683..6294cb71 100644
--- a/bchain/coins/eth/ethparser_test.go
+++ b/bchain/coins/eth/ethparser_test.go
@@ -199,6 +199,8 @@ func TestEthereumParser_UnpackTx(t *testing.T) {
 				t.Errorf("EthereumParser.UnpackTx() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
+			// CoinSpecificData are not set in want struct
+			got.CoinSpecificData = nil
 			// DeepEqual compares empty nil slices as not equal
 			if fmt.Sprint(got) != fmt.Sprint(tt.want) {
 				t.Errorf("EthereumParser.UnpackTx() got = %+v, want %+v", got, tt.want)
diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go
index 468aee97..5bb486ad 100644
--- a/bchain/coins/eth/ethrpc.go
+++ b/bchain/coins/eth/ethrpc.go
@@ -447,7 +447,7 @@ func (b *EthereumRPC) GetBlock(hash string, height uint32) (*bchain.Block, error
 	// TODO - get ERC20 events
 	btxs := make([]bchain.Tx, len(body.Transactions))
 	for i, tx := range body.Transactions {
-		btx, err := b.Parser.ethTxToTx(&tx, nil, int64(head.Time.Uint64()), uint32(bbh.Confirmations))
+		btx, err := b.Parser.ethTxToTx(&tx, nil, int64(head.Time.Uint64()), uint32(bbh.Confirmations), false)
 		if err != nil {
 			return nil, errors.Annotatef(err, "hash %v, height %v, txid %v", hash, height, tx.Hash.String())
 		}
@@ -493,7 +493,7 @@ func (b *EthereumRPC) GetTransaction(txid string) (*bchain.Tx, error) {
 	var btx *bchain.Tx
 	if tx.BlockNumber == "" {
 		// mempool tx
-		btx, err = b.Parser.ethTxToTx(tx, nil, 0, 0)
+		btx, err = b.Parser.ethTxToTx(tx, nil, 0, 0, true)
 		if err != nil {
 			return nil, errors.Annotatef(err, "txid %v", txid)
 		}
@@ -516,7 +516,7 @@ func (b *EthereumRPC) GetTransaction(txid string) (*bchain.Tx, error) {
 		if err != nil {
 			return nil, errors.Annotatef(err, "txid %v", txid)
 		}
-		btx, err = b.Parser.ethTxToTx(tx, &receipt, h.Time.Int64(), confirmations)
+		btx, err = b.Parser.ethTxToTx(tx, &receipt, h.Time.Int64(), confirmations, true)
 		if err != nil {
 			return nil, errors.Annotatef(err, "txid %v", txid)
 		}
@@ -525,17 +525,20 @@ func (b *EthereumRPC) GetTransaction(txid string) (*bchain.Tx, error) {
 }
 
 // GetTransactionSpecific returns json as returned by backend, with all coin specific data
-func (b *EthereumRPC) GetTransactionSpecific(txid string) (json.RawMessage, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
-	defer cancel()
-	var tx json.RawMessage
-	err := b.rpc.CallContext(ctx, &tx, "eth_getTransactionByHash", ethcommon.HexToHash(txid))
-	if err != nil {
-		return nil, err
-	} else if tx == nil {
-		return nil, ethereum.NotFound
+func (b *EthereumRPC) GetTransactionSpecific(tx *bchain.Tx) (json.RawMessage, error) {
+	csd, ok := tx.CoinSpecificData.(completeTransaction)
+	if !ok {
+		ntx, err := b.GetTransaction(tx.Txid)
+		if err != nil {
+			return nil, err
+		}
+		csd, ok = ntx.CoinSpecificData.(completeTransaction)
+		if !ok {
+			return nil, errors.New("Cannot get CoinSpecificData")
+		}
 	}
-	return tx, nil
+	m, err := json.Marshal(&csd)
+	return json.RawMessage(m), err
 }
 
 type rpcMempoolBlock struct {
diff --git a/bchain/types.go b/bchain/types.go
index 327dfdcd..5053ff4d 100644
--- a/bchain/types.go
+++ b/bchain/types.go
@@ -75,9 +75,10 @@ type Tx struct {
 	Vin      []Vin  `json:"vin"`
 	Vout     []Vout `json:"vout"`
 	// BlockHash     string `json:"blockhash,omitempty"`
-	Confirmations uint32 `json:"confirmations,omitempty"`
-	Time          int64  `json:"time,omitempty"`
-	Blocktime     int64  `json:"blocktime,omitempty"`
+	Confirmations    uint32      `json:"confirmations,omitempty"`
+	Time             int64       `json:"time,omitempty"`
+	Blocktime        int64       `json:"blocktime,omitempty"`
+	CoinSpecificData interface{} `json:"-"`
 }
 
 // Block is block header and list of transactions
@@ -185,7 +186,7 @@ type BlockChain interface {
 	GetMempool() ([]string, error)
 	GetTransaction(txid string) (*Tx, error)
 	GetTransactionForMempool(txid string) (*Tx, error)
-	GetTransactionSpecific(txid string) (json.RawMessage, error)
+	GetTransactionSpecific(tx *Tx) (json.RawMessage, error)
 	EstimateSmartFee(blocks int, conservative bool) (big.Int, error)
 	EstimateFee(blocks int) (big.Int, error)
 	SendRawTransaction(tx string) (string, error)
diff --git a/server/public.go b/server/public.go
index e581d81f..de61b9fc 100644
--- a/server/public.go
+++ b/server/public.go
@@ -335,7 +335,6 @@ type TemplateData struct {
 	Address          *api.Address
 	AddrStr          string
 	Tx               *api.Tx
-	TxSpecific       json.RawMessage
 	Error            *api.APIError
 	Blocks           *api.Blocks
 	Block            *api.Block
@@ -391,23 +390,17 @@ func setTxToTemplateData(td *TemplateData, tx *api.Tx) *TemplateData {
 
 func (s *PublicServer) explorerTx(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) {
 	var tx *api.Tx
-	var txSpecific json.RawMessage
 	var err error
 	s.metrics.ExplorerViews.With(common.Labels{"action": "tx"}).Inc()
 	if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 {
 		txid := r.URL.Path[i+1:]
-		tx, err = s.api.GetTransaction(txid, false)
-		if err != nil {
-			return errorTpl, nil, err
-		}
-		txSpecific, err = s.chain.GetTransactionSpecific(txid)
+		tx, err = s.api.GetTransaction(txid, false, true)
 		if err != nil {
 			return errorTpl, nil, err
 		}
 	}
 	data := s.newTemplateData()
 	data.Tx = tx
-	data.TxSpecific = txSpecific
 	return txTpl, data, nil
 }
 
@@ -520,7 +513,7 @@ func (s *PublicServer) explorerSearch(w http.ResponseWriter, r *http.Request) (t
 			http.Redirect(w, r, joinURL("/block/", block.Hash), 302)
 			return noTpl, nil, nil
 		}
-		tx, err = s.api.GetTransaction(q, false)
+		tx, err = s.api.GetTransaction(q, false, false)
 		if err == nil {
 			http.Redirect(w, r, joinURL("/tx/", tx.Txid), 302)
 			return noTpl, nil, nil
@@ -655,7 +648,7 @@ func (s *PublicServer) apiTx(r *http.Request) (interface{}, error) {
 				return nil, api.NewAPIError("Parameter 'spending' cannot be converted to boolean", true)
 			}
 		}
-		tx, err = s.api.GetTransaction(txid, spendingTxs)
+		tx, err = s.api.GetTransaction(txid, spendingTxs, false)
 	}
 	return tx, err
 }
@@ -666,7 +659,7 @@ func (s *PublicServer) apiTxSpecific(r *http.Request) (interface{}, error) {
 	s.metrics.ExplorerViews.With(common.Labels{"action": "api-tx-specific"}).Inc()
 	if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 {
 		txid := r.URL.Path[i+1:]
-		tx, err = s.chain.GetTransactionSpecific(txid)
+		tx, err = s.chain.GetTransactionSpecific(&bchain.Tx{Txid: txid})
 	}
 	return tx, err
 }
diff --git a/server/socketio.go b/server/socketio.go
index 2cdd1361..387dd256 100644
--- a/server/socketio.go
+++ b/server/socketio.go
@@ -387,7 +387,7 @@ func (s *SocketIoServer) getAddressHistory(addr []string, opts *addrOpts) (res r
 		to = opts.To
 	}
 	for txi := opts.From; txi < to; txi++ {
-		tx, err := s.api.GetTransaction(txids[txi], false)
+		tx, err := s.api.GetTransaction(txids[txi], false, false)
 		if err != nil {
 			return res, err
 		}
@@ -627,7 +627,7 @@ type resultGetDetailedTransaction struct {
 }
 
 func (s *SocketIoServer) getDetailedTransaction(txid string) (res resultGetDetailedTransaction, err error) {
-	tx, err := s.api.GetTransaction(txid, false)
+	tx, err := s.api.GetTransaction(txid, false, false)
 	if err != nil {
 		return res, err
 	}
diff --git a/static/templates/tx.html b/static/templates/tx.html
index b04983ba..22aca6a3 100644
--- a/static/templates/tx.html
+++ b/static/templates/tx.html
@@ -1,4 +1,4 @@
-{{define "specific"}}{{$cs := .CoinShortcut}}{{$tx := .Tx}}{{$txSpecific := .TxSpecific}}
+{{define "specific"}}{{$cs := .CoinShortcut}}{{$tx := .Tx}}
 <h1>Transaction</h1>
 <div class="alert alert-data ellipsis">
     <span class="data">{{$tx.Txid}}</span>
@@ -47,7 +47,7 @@
         <pre id="txSpecific"></pre>
     </div>
     <script type="text/javascript">
-        txSpecific = {{ $txSpecific }};
+        txSpecific = {{$tx.CoinSpecificData}};
         function syntaxHighlight(json) {
             json = JSON.stringify(json, undefined, 2);
             json = json.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
diff --git a/tests/dbtestdata/fakechain.go b/tests/dbtestdata/fakechain.go
index b7ef7cba..e379badf 100644
--- a/tests/dbtestdata/fakechain.go
+++ b/tests/dbtestdata/fakechain.go
@@ -141,8 +141,8 @@ func (c *fakeBlockChain) GetTransaction(txid string) (v *bchain.Tx, err error) {
 	return nil, errors.New("Not found")
 }
 
-func (c *fakeBlockChain) GetTransactionSpecific(txid string) (v json.RawMessage, err error) {
-	tx, err := c.GetTransaction(txid)
+func (c *fakeBlockChain) GetTransactionSpecific(tx *bchain.Tx) (v json.RawMessage, err error) {
+	tx, err = c.GetTransaction(tx.Txid)
 	if err != nil {
 		return nil, err
 	}