Implement GetAddressForXpub and xpub explorer view - WIP

xpub
Martin Boehm 2019-01-30 17:56:15 +01:00
parent b670b4fede
commit 225830d3e9
5 changed files with 451 additions and 1 deletions

View File

@ -5,6 +5,7 @@ import (
"blockbook/common"
"blockbook/db"
"encoding/json"
"errors"
"math/big"
"time"
)
@ -27,6 +28,9 @@ const (
TxHistory
)
// ErrUnsupportedXpub is returned when coin type does not support xpub address derivation or provided string is not an xpub
var ErrUnsupportedXpub = errors.New("XPUB not supported")
// APIError extends error by information if the error details should be returned to the end user
type APIError struct {
Text string
@ -123,6 +127,9 @@ type TokenType string
// ERC20TokenType is Ethereum ERC20 token
const ERC20TokenType TokenType = "ERC20"
// XPUBAddressTokenType is address derived from xpub
const XPUBAddressTokenType TokenType = "XPUBAddress"
// Token contains info about tokens held by an address
type Token struct {
Type TokenType `json:"type"`

272
api/xpub.go 100644
View File

@ -0,0 +1,272 @@
package api
import (
"blockbook/bchain"
"blockbook/db"
"fmt"
"math/big"
"sync"
"time"
"github.com/golang/glog"
"github.com/juju/errors"
)
const xpubLen = 111
const derivedAddressesBlock = 20
var cachedXpubs = make(map[string]*xpubData)
var cachedXpubsMux sync.Mutex
type txHeight struct {
txid string
height uint32
addrIndex uint32
}
type xpubAddress struct {
addrDesc bchain.AddressDescriptor
balance *db.AddrBalance
bottomHeight uint32
}
type xpubData struct {
dataHeight uint32
dataHash string
txs uint32
sentSat big.Int
balanceSat big.Int
addresses []xpubAddress
changeAddresses []xpubAddress
txids []txHeight
}
func (w *Worker) getAddressTxHeights(addrDesc bchain.AddressDescriptor, addrIndex uint32, mempool bool, filter *AddressFilter, maxResults int) ([]txHeight, error) {
var err error
txHeights := make([]txHeight, 0, 4)
var callback db.GetTransactionsCallback
if filter.Vout == AddressFilterVoutOff {
callback = func(txid string, height uint32, indexes []int32) error {
txHeights = append(txHeights, txHeight{txid, height, addrIndex})
// take all txs in the last found block even if it exceeds maxResults
if len(txHeights) >= maxResults && txHeights[len(txHeights)-1].height != height {
return &db.StopIteration{}
}
return nil
}
} else {
callback = func(txid string, height uint32, indexes []int32) error {
for _, index := range indexes {
vout := index
if vout < 0 {
vout = ^vout
}
if (filter.Vout == AddressFilterVoutInputs && index < 0) ||
(filter.Vout == AddressFilterVoutOutputs && index >= 0) ||
(vout == int32(filter.Vout)) {
txHeights = append(txHeights, txHeight{txid, height, addrIndex})
if len(txHeights) >= maxResults {
return &db.StopIteration{}
}
break
}
}
return nil
}
}
if mempool {
uniqueTxs := make(map[string]struct{})
o, err := w.chain.GetMempoolTransactionsForAddrDesc(addrDesc)
if err != nil {
return nil, err
}
for _, m := range o {
if _, found := uniqueTxs[m.Txid]; !found {
l := len(txHeights)
callback(m.Txid, 0, []int32{m.Vout})
if len(txHeights) > l {
uniqueTxs[m.Txid] = struct{}{}
}
}
}
} else {
to := filter.ToHeight
if to == 0 {
to = ^uint32(0)
}
err = w.db.GetAddrDescTransactions(addrDesc, filter.FromHeight, to, callback)
if err != nil {
return nil, err
}
}
return txHeights, nil
}
func (w *Worker) derivedAddressBalance(data *xpubData, ad *xpubAddress) (bool, error) {
var err error
if ad.balance, err = w.db.GetAddrDescBalance(ad.addrDesc); err != nil {
return false, err
}
if ad.balance != nil {
data.txs += ad.balance.Txs
data.sentSat.Add(&data.sentSat, &ad.balance.SentSat)
data.balanceSat.Add(&data.balanceSat, &ad.balance.BalanceSat)
return true, nil
}
return false, nil
}
func (w *Worker) tokenFromXpubAddress(ad *xpubAddress, changeIndex int, index int) Token {
a, _, _ := w.chainParser.GetAddressesFromAddrDesc(ad.addrDesc)
var address string
if len(a) > 0 {
address = a[0]
}
return Token{
Type: XPUBAddressTokenType,
Name: address,
Decimals: w.chainParser.AmountDecimals(),
BalanceSat: (*Amount)(&ad.balance.BalanceSat),
Transfers: int(ad.balance.Txs),
Contract: fmt.Sprintf("%d/%d", changeIndex, index),
}
}
// GetAddressForXpub computes address value and gets transactions for given address
func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option GetAddressOption, filter *AddressFilter) (*Address, error) {
if w.chainType != bchain.ChainBitcoinType || len(xpub) != xpubLen {
return nil, ErrUnsupportedXpub
}
start := time.Now()
var processedHash string
cachedXpubsMux.Lock()
data, found := cachedXpubs[xpub]
cachedXpubsMux.Unlock()
// to load all data for xpub may take some time, perform it in a loop to process a possible new block
for {
bestheight, besthash, err := w.db.GetBestBlock()
if err != nil {
return nil, errors.Annotatef(err, "GetBestBlock")
}
if besthash == processedHash {
break
}
fork := false
if !found {
data = &xpubData{}
} else {
hash, err := w.db.GetBlockHash(data.dataHeight)
if err != nil {
return nil, err
}
if hash != data.dataHash {
// in case of for reset all cached txids
fork = true
data.txids = nil
}
}
processedHash = besthash
if data.dataHeight < bestheight {
data.dataHeight = bestheight
data.dataHash = besthash
// rescan known addresses
lastUsed := 0
for i := range data.addresses {
ad := &data.addresses[i]
if fork {
ad.bottomHeight = 0
}
used, err := w.derivedAddressBalance(data, ad)
if err != nil {
return nil, err
}
if used {
lastUsed = i
}
}
// derive new addresses as necessary
missing := len(data.addresses) - lastUsed
for missing < derivedAddressesBlock {
from := len(data.addresses)
descriptors, err := w.chainParser.DeriveAddressDescriptorsFromTo(xpub, 0, uint32(from), uint32(from+derivedAddressesBlock-missing))
if err != nil {
return nil, err
}
for i, a := range descriptors {
ad := xpubAddress{addrDesc: a}
used, err := w.derivedAddressBalance(data, &ad)
if err != nil {
return nil, err
}
if used {
lastUsed = i + from
}
data.addresses = append(data.addresses, ad)
}
missing = len(data.addresses) - lastUsed
}
// check and generate change addresses
ca := data.changeAddresses
data.changeAddresses = make([]xpubAddress, len(data.addresses))
copy(data.changeAddresses, ca)
changeIndexes := []uint32{}
for i, ad := range data.addresses {
if ad.balance != nil {
if data.changeAddresses[i].addrDesc == nil {
changeIndexes = append(changeIndexes, uint32(i))
} else {
_, err := w.derivedAddressBalance(data, &ad)
if err != nil {
return nil, err
}
}
}
}
if len(changeIndexes) > 0 {
descriptors, err := w.chainParser.DeriveAddressDescriptors(xpub, 1, changeIndexes)
if err != nil {
return nil, err
}
for i, a := range descriptors {
ad := &data.changeAddresses[changeIndexes[i]]
ad.addrDesc = a
_, err := w.derivedAddressBalance(data, ad)
if err != nil {
return nil, err
}
}
}
}
}
cachedXpubsMux.Lock()
cachedXpubs[xpub] = data
cachedXpubsMux.Unlock()
tokens := make([]Token, 0, 4)
for i, ad := range data.addresses {
if ad.balance != nil {
tokens = append(tokens, w.tokenFromXpubAddress(&ad, 0, i))
}
if data.changeAddresses[i].balance != nil {
tokens = append(tokens, w.tokenFromXpubAddress(&data.changeAddresses[i], 1, i))
}
}
var totalReceived big.Int
totalReceived.Add(&data.balanceSat, &data.sentSat)
addr := Address{
// Paging: pg,
AddrStr: xpub,
BalanceSat: (*Amount)(&data.balanceSat),
TotalReceivedSat: (*Amount)(&totalReceived),
TotalSentSat: (*Amount)(&data.sentSat),
Txs: int(data.txs),
// UnconfirmedBalanceSat: (*Amount)(&uBalSat),
// UnconfirmedTxs: len(txm),
// Transactions: txs,
// Txids: txids,
Tokens: tokens,
// Erc20Contract: erc20c,
// Nonce: nonce,
}
glog.Info("GetAddressForXpub ", xpub[:10], ", ", len(data.addresses), " derived addresses, ", data.txs, " total txs finished in ", time.Since(start))
return &addr, nil
}

View File

@ -131,6 +131,7 @@ func (s *PublicServer) ConnectFullPublicInterface() {
// internal explorer handlers
serveMux.HandleFunc(path+"tx/", s.htmlTemplateHandler(s.explorerTx))
serveMux.HandleFunc(path+"address/", s.htmlTemplateHandler(s.explorerAddress))
serveMux.HandleFunc(path+"xpub/", s.htmlTemplateHandler(s.explorerXpub))
serveMux.HandleFunc(path+"search/", s.htmlTemplateHandler(s.explorerSearch))
serveMux.HandleFunc(path+"blocks", s.htmlTemplateHandler(s.explorerBlocks))
serveMux.HandleFunc(path+"block/", s.htmlTemplateHandler(s.explorerBlock))
@ -165,6 +166,7 @@ func (s *PublicServer) ConnectFullPublicInterface() {
serveMux.HandleFunc(path+"api/tx-specific/", s.jsonHandler(s.apiTxSpecific, apiDefault))
serveMux.HandleFunc(path+"api/tx/", s.jsonHandler(s.apiTx, apiDefault))
serveMux.HandleFunc(path+"api/address/", s.jsonHandler(s.apiAddress, apiDefault))
serveMux.HandleFunc(path+"api/xpub/", s.jsonHandler(s.apiXpub, apiDefault))
serveMux.HandleFunc(path+"api/utxo/", s.jsonHandler(s.apiAddressUtxo, apiDefault))
serveMux.HandleFunc(path+"api/block/", s.jsonHandler(s.apiBlock, apiDefault))
serveMux.HandleFunc(path+"api/sendtx/", s.jsonHandler(s.apiSendTx, apiDefault))
@ -174,6 +176,7 @@ func (s *PublicServer) ConnectFullPublicInterface() {
serveMux.HandleFunc(path+"api/v2/tx-specific/", s.jsonHandler(s.apiTxSpecific, apiV2))
serveMux.HandleFunc(path+"api/v2/tx/", s.jsonHandler(s.apiTx, apiV2))
serveMux.HandleFunc(path+"api/v2/address/", s.jsonHandler(s.apiAddress, apiV2))
serveMux.HandleFunc(path+"api/v2/xpub/", s.jsonHandler(s.apiXpub, apiV2))
serveMux.HandleFunc(path+"api/v2/utxo/", s.jsonHandler(s.apiAddressUtxo, apiV2))
serveMux.HandleFunc(path+"api/v2/block/", s.jsonHandler(s.apiBlock, apiV2))
serveMux.HandleFunc(path+"api/v2/sendtx/", s.jsonHandler(s.apiSendTx, apiV2))
@ -372,6 +375,7 @@ const (
indexTpl
txTpl
addressTpl
xpubTpl
blocksTpl
blockTpl
sendTransactionTpl
@ -465,6 +469,7 @@ func (s *PublicServer) parseTemplates() []*template.Template {
t[addressTpl] = createTemplate("./static/templates/address.html", "./static/templates/txdetail.html", "./static/templates/paging.html", "./static/templates/base.html")
t[blockTpl] = createTemplate("./static/templates/block.html", "./static/templates/txdetail.html", "./static/templates/paging.html", "./static/templates/base.html")
}
t[xpubTpl] = createTemplate("./static/templates/xpub.html", "./static/templates/txdetail.html", "./static/templates/paging.html", "./static/templates/base.html")
return t
}
@ -577,6 +582,48 @@ func (s *PublicServer) explorerAddress(w http.ResponseWriter, r *http.Request) (
return addressTpl, data, nil
}
func (s *PublicServer) explorerXpub(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) {
var address *api.Address
var filter string
var fn = api.AddressFilterVoutOff
var err error
s.metrics.ExplorerViews.With(common.Labels{"action": "xpub"}).Inc()
if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 {
page, ec := strconv.Atoi(r.URL.Query().Get("page"))
if ec != nil {
page = 0
}
filter = r.URL.Query().Get("filter")
if len(filter) > 0 {
if filter == "inputs" {
fn = api.AddressFilterVoutInputs
} else if filter == "outputs" {
fn = api.AddressFilterVoutOutputs
} else {
fn, ec = strconv.Atoi(filter)
if ec != nil || fn < 0 {
filter = ""
fn = api.AddressFilterVoutOff
}
}
}
address, err = s.api.GetAddressForXpub(r.URL.Path[i+1:], page, txsOnPage, api.TxHistoryLight, &api.AddressFilter{Vout: fn})
if err != nil {
return errorTpl, nil, err
}
}
data := s.newTemplateData()
data.AddrStr = address.AddrStr
data.Address = address
data.Page = address.Page
data.PagingRange, data.PrevPage, data.NextPage = getPagingRange(address.Page, address.TotalPages)
if filter != "" {
data.PageParams = template.URL("&filter=" + filter)
data.Address.Filter = filter
}
return xpubTpl, data, nil
}
func (s *PublicServer) explorerBlocks(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) {
var blocks *api.Blocks
var err error
@ -638,6 +685,11 @@ func (s *PublicServer) explorerSearch(w http.ResponseWriter, r *http.Request) (t
var err error
s.metrics.ExplorerViews.With(common.Labels{"action": "search"}).Inc()
if len(q) > 0 {
address, err = s.api.GetAddressForXpub(q, 0, 1, api.Basic, &api.AddressFilter{Vout: api.AddressFilterVoutOff})
if err == nil {
http.Redirect(w, r, joinURL("/xpub/", address.AddrStr), 302)
return noTpl, nil, nil
}
block, err = s.api.GetBlock(q, 0, 1)
if err == nil {
http.Redirect(w, r, joinURL("/block/", block.Hash), 302)
@ -818,6 +870,23 @@ func (s *PublicServer) apiAddress(r *http.Request, apiVersion int) (interface{},
return address, err
}
func (s *PublicServer) apiXpub(r *http.Request, apiVersion int) (interface{}, error) {
var address *api.Address
var err error
s.metrics.ExplorerViews.With(common.Labels{"action": "api-xpub"}).Inc()
if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 {
page, ec := strconv.Atoi(r.URL.Query().Get("page"))
if ec != nil {
page = 0
}
address, err = s.api.GetAddressForXpub(r.URL.Path[i+1:], page, txsInAPI, api.TxidHistory, &api.AddressFilter{Vout: api.AddressFilterVoutOff})
if err == nil && apiVersion == apiV1 {
return s.api.AddressToV1(address), nil
}
}
return address, err
}
func (s *PublicServer) apiAddressUtxo(r *http.Request, apiVersion int) (interface{}, error) {
var utxo []api.AddressUtxo
var err error

View File

@ -42,7 +42,7 @@
</ul>
<span class="d-none ml-md-auto d-md-flex navbar-form navbar-left">
<form id="search" action="/search" method="get">
<input name="q" type="text" class="form-control" placeholder="Search for block, transaction or address" focus="true">
<input name="q" type="text" class="form-control" placeholder="Search for block, transaction, address or xpub" focus="true">
</form>
</span>
{{- end -}}

View File

@ -0,0 +1,102 @@
{{define "specific"}}{{$cs := .CoinShortcut}}{{$addr := .Address}}{{$data := .}}
<h1>XPUB <small class="text-muted">{{formatAmount $addr.BalanceSat}} {{$cs}}</small>
</h1>
<div class="alert alert-data ellipsis">
<span class="data">{{$addr.AddrStr}}</span>
</div>
<h3>Confirmed</h3>
<div class="data-div row">
<div class="col-md-10">
<table class="table data-table">
<tbody>
<tr>
<td style="width: 25%;">Total Received</td>
<td class="data">{{formatAmount $addr.TotalReceivedSat}} {{$cs}}</td>
</tr>
<tr>
<td>Total Sent</td>
<td class="data">{{formatAmount $addr.TotalSentSat}} {{$cs}}</td>
</tr>
<tr>
<td>Final Balance</td>
<td class="data">{{formatAmount $addr.BalanceSat}} {{$cs}}</td>
</tr>
<tr>
<td>No. Transactions</td>
<td class="data">{{$addr.Txs}}</td>
</tr>
{{- if $addr.Tokens -}}
<tr>
<td>XPUB addresses</td>
<td style="padding: 0;">
<table class="table data-table">
<tbody>
<tr>
<th style="width: 50%;">Address</th>
<th>Balance</th>
<th style="width: 10%;">Txs</th>
<th style="width: 10%;">Path</th>
</tr>
{{- range $t := $addr.Tokens -}}
<tr>
<td class="data ellipsis"><a href="/address/{{$t.Name}}">{{$t.Name}}</a></td>
<td class="data">{{formatAmount $t.BalanceSat}} {{$cs}}</td>
<td class="data">{{$t.Transfers}}</td>
<td class="data">{{$t.Contract}}</td>
</tr>
{{- end -}}
</tbody>
</table>
</td>
</tr>
{{- end -}}
</tbody>
</table>
</div>
<div class="col-md-2">
<div id="qrcode" style="width: 160px; height: 160px;"></div>
<script type="text/javascript" src="/static/js/qrcode.min.js"></script>
<script type="text/javascript">
new QRCode(document.getElementById("qrcode"), { text: "{{$addr.AddrStr}}", width: 160, height: 160 });
</script>
</div>
</div>
{{- if $addr.UnconfirmedTxs -}}
<h3>Unconfirmed</h3>
<div class="data-div">
<table class="table data-table">
<tbody>
<tr>
<td style="width: 25%;">Unconfirmed Balance</td>
<td class="data">{{formatAmount $addr.UnconfirmedBalanceSat}} {{$cs}}</td>
</tr>
<tr>
<td>No. Transactions</td>
<td class="data">{{$addr.UnconfirmedTxs}}</td>
</tr>
</tbody>
</table>
</div>
{{- end}}{{if or $addr.Transactions $addr.Filter -}}
<div class="row h-container">
<h3 class="col-md-3">Transactions</h3>
<select class="col-md-2" style="background-color: #eaeaea;" onchange="self.location='?filter='+options[selectedIndex].value">
<option>All</option>
<option {{if eq $addr.Filter "inputs" -}} selected{{end}} value="inputs">Inputs</option>
<option {{if eq $addr.Filter "outputs" -}} selected{{end}} value="outputs">Outputs</option>
{{- if $addr.Tokens -}}
<option {{if eq $addr.Filter "0" -}} selected{{end}} value="0">Non-contract</option>
{{- range $t := $addr.Tokens -}}
<option {{if eq $addr.Filter $t.ContractIndex -}} selected{{end}} value="{{$t.ContractIndex}}">{{$t.Name}}</option>
{{- end -}}
{{- end -}}
</select>
<div class="col-md-7">
<nav>{{template "paging" $data}}</nav>
</div>
</div>
<div class="data-div">
{{- range $tx := $addr.Transactions}}{{$data := setTxToTemplateData $data $tx}}{{template "txdetail" $data}}{{end -}}
</div>
<nav>{{template "paging" $data }}</nav>
{{end}}{{end}}