commit 9ada001e3c785682985a3f6cbc02b5013ae2d805 Author: Jan Pochyla Date: Mon Aug 28 17:50:57 2017 +0200 initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..a3fb6911 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +vendor/ +blockbook +notes.txt diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 00000000..61865709 --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,51 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + branch = "master" + name = "github.com/bsm/go-vlq" + packages = ["."] + revision = "ec6e8d4f5f4ec0f6e808ffc7f4dcc7516d4d7d49" + +[[projects]] + branch = "master" + name = "github.com/btcsuite/btcd" + packages = ["btcec","chaincfg","chaincfg/chainhash","txscript","wire"] + revision = "a1d1ea70dd212a440beb9caa4b766a58d1ed0254" + +[[projects]] + branch = "master" + name = "github.com/btcsuite/btclog" + packages = ["."] + revision = "84c8d2346e9fc8c7b947e243b9c24e6df9fd206a" + +[[projects]] + branch = "master" + name = "github.com/btcsuite/btcutil" + packages = [".","base58","bech32"] + revision = "501929d3d046174c3d39f0ea54ece471aa17238c" + +[[projects]] + branch = "master" + name = "github.com/hashicorp/golang-lru" + packages = [".","simplelru"] + revision = "0a025b7e63adc15a622f29b0b2c4c3848243bbf6" + +[[projects]] + branch = "master" + name = "github.com/tecbot/gorocksdb" + packages = ["."] + revision = "b9cb0d30eca790f5bcf524dd3beb715d6c8d1923" + +[[projects]] + branch = "master" + name = "golang.org/x/crypto" + packages = ["ripemd160"] + revision = "81e90905daefcd6fd217b62423c0908922eadb30" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "3bb30a1d65e170c9b0f282efa56af8096dc74f149744743dddd27f26e1880612" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 00000000..f92435a9 --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,42 @@ + +# Gopkg.toml example +# +# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" + + +[[constraint]] + branch = "master" + name = "github.com/bsm/go-vlq" + +[[constraint]] + branch = "master" + name = "github.com/btcsuite/btcd" + +[[constraint]] + branch = "master" + name = "github.com/btcsuite/btcutil" + +[[constraint]] + branch = "master" + name = "github.com/hashicorp/golang-lru" + +[[constraint]] + branch = "master" + name = "github.com/tecbot/gorocksdb" diff --git a/bitcoinrpc.go b/bitcoinrpc.go new file mode 100644 index 00000000..02604488 --- /dev/null +++ b/bitcoinrpc.go @@ -0,0 +1,139 @@ +package main + +import ( + "encoding/hex" + "log" + "net/http" + "time" + + lru "github.com/hashicorp/golang-lru" +) + +// BitcoinRPC is an interface to JSON-RPC bitcoind service. +type BitcoinRPC struct { + client JSONRPC + Parser BlockParser + txCache *lru.Cache +} + +// NewBitcoinRPC returns new BitcoinRPC instance. +func NewBitcoinRPC(url string, user string, password string, timeout time.Duration) *BitcoinRPC { + return &BitcoinRPC{ + client: JSONRPC{ + Client: http.Client{Timeout: timeout}, + URL: url, + User: user, + Password: password, + }, + } +} + +// EnableCache turns on LRU caching for transaction lookups. +func (b *BitcoinRPC) EnableCache(size int) error { + c, err := lru.New(size) + if err != nil { + return err + } + b.txCache = c + return nil +} + +// ClearCache purges the cache used for transaction results. +func (b *BitcoinRPC) ClearCache() { + if b.txCache != nil { + b.txCache.Purge() + } +} + +// GetBlock returns information about the block with the given hash. +func (b *BitcoinRPC) GetBlock(hash string, height uint32) (block *Block, err error) { + if b.Parser != nil { + return b.GetBlockAndParse(hash, height) + } else { + return b.GetParsedBlock(hash) + } +} + +// GetBlockAndParse returns information about the block with the given hash. +// +// It downloads raw block and parses it in-process. +func (b *BitcoinRPC) GetBlockAndParse(hash string, height uint32) (block *Block, err error) { + log.Printf("rpc: getblock (verbose=false) %v", hash) + var raw string + err = b.client.Call("getblock", &raw, hash, false) // verbose=false + if err != nil { + return + } + data, err := hex.DecodeString(raw) + if err != nil { + return + } + block, err = b.Parser.ParseBlock(data) + if err == nil { + block.Hash = hash + block.Height = height + } + return +} + +// GetParsedBlock returns information about the block with the given hash. +// +// It downloads parsed block with transaction IDs and then looks them up, +// one by one. +func (b *BitcoinRPC) GetParsedBlock(hash string) (block *Block, err error) { + log.Printf("rpc: getblock (verbose=true) %v", hash) + block = &Block{} + err = b.client.Call("getblock", block, hash, true) // verbose=true + if err != nil { + return + } + for _, txid := range block.Txids { + tx, err := b.GetTransaction(txid) + if err != nil { + return nil, err + } + block.Txs = append(block.Txs, tx) + } + return +} + +// GetBlockHash returns hash of block in best-block-chain at given height. +func (b *BitcoinRPC) GetBlockHash(height uint32) (hash string, err error) { + log.Printf("rpc: getblockhash %v", height) + err = b.client.Call("getblockhash", &hash, height) + return +} + +// GetBlockCount returns the number of blocks in the longest chain. +func (b *BitcoinRPC) GetBlockCount() (count uint32, err error) { + log.Printf("rpc: getblockcount") + err = b.client.Call("getblockcount", &count) + return +} + +// GetRawTransaction returns the number of blocks in the longest chain. If the +// transaction cache is turned on, returned RawTx.Confirmations is stale. +func (b *BitcoinRPC) GetTransaction(txid string) (tx *Tx, err error) { + if b.txCache != nil { + if cachedTx, ok := b.txCache.Get(txid); ok { + tx = cachedTx.(*Tx) + return + } + } + log.Printf("rpc: getrawtransaction %v", txid) + tx = &Tx{} + err = b.client.Call("getrawtransaction", tx, txid, true) // verbose = true + if b.txCache != nil { + b.txCache.Add(txid, tx) + } + return +} + +// GetOutpointAddresses returns all unique addresses from given transaction output. +func (b *BitcoinRPC) GetOutpointAddresses(txid string, vout uint32) ([]string, error) { + tx, err := b.GetTransaction(txid) + if err != nil { + return nil, err + } + return tx.Vout[vout].ScriptPubKey.Addresses, nil +} diff --git a/bitcoinwire.go b/bitcoinwire.go new file mode 100644 index 00000000..c79c7363 --- /dev/null +++ b/bitcoinwire.go @@ -0,0 +1,88 @@ +package main + +import ( + "bytes" + "encoding/hex" + + "github.com/btcsuite/btcd/chaincfg" + "github.com/btcsuite/btcd/txscript" + "github.com/btcsuite/btcd/wire" +) + +func GetChainParams() []*chaincfg.Params { + return []*chaincfg.Params{ + &chaincfg.MainNetParams, + &chaincfg.RegressionNetParams, + &chaincfg.TestNet3Params, + &chaincfg.SimNetParams, + } +} + +type BitcoinBlockParser struct { + Params *chaincfg.Params +} + +func (p *BitcoinBlockParser) parseOutputScript(b []byte) (addrs []string, err error) { + _, addresses, _, err := txscript.ExtractPkScriptAddrs(b, p.Params) + for _, a := range addresses { + addr := a.EncodeAddress() + addrs = append(addrs, addr) + } + return addrs, err +} + +func (p *BitcoinBlockParser) ParseBlock(b []byte) (*Block, error) { + w := wire.MsgBlock{} + r := bytes.NewReader(b) + if err := w.DeserializeNoWitness(r); err != nil { + return nil, err + } + + block := &Block{} + for _, t := range w.Transactions { + tx := &Tx{ + Txid: t.TxHash().String(), + Version: t.Version, + LockTime: t.LockTime, + Vin: make([]Vin, len(t.TxIn)), + Vout: make([]Vout, len(t.TxOut)), + // missing: BlockHash, + // missing: Confirmations, + // missing: Time, + // missing: Blocktime, + } + for i, in := range t.TxIn { + s := ScriptSig{ + Hex: hex.EncodeToString(in.SignatureScript), + // missing: Asm, + } + tx.Vin[i] = Vin{ + Coinbase: "_", + Txid: in.PreviousOutPoint.Hash.String(), + Vout: in.PreviousOutPoint.Index, + Sequence: in.Sequence, + ScriptSig: s, + } + } + for i, out := range t.TxOut { + addrs, err := p.parseOutputScript(out.PkScript) + if err != nil { + return nil, err + } + s := ScriptPubKey{ + Hex: hex.EncodeToString(out.PkScript), + Addresses: addrs, + // missing: Asm, + // missing: Type, + } + tx.Vout[i] = Vout{ + Value: float64(out.Value), + N: uint32(i), + ScriptPubKey: s, + } + } + block.Txs = append(block.Txs, tx) + } + + return block, nil +} diff --git a/blockbook.go b/blockbook.go new file mode 100644 index 00000000..c3bb4884 --- /dev/null +++ b/blockbook.go @@ -0,0 +1,173 @@ +package main + +import ( + "flag" + "log" + "time" +) + +type BlockParser interface { + ParseBlock(b []byte) (*Block, error) +} + +type BlockOracle interface { + GetBlockHash(height uint32) (string, error) + GetBlock(hash string, height uint32) (*Block, error) +} + +type OutpointAddressOracle interface { + GetOutpointAddresses(txid string, vout uint32) ([]string, error) +} + +type AddressTransactionOracle interface { + GetAddressTransactions(address string, lower uint32, higher uint32, fn func(txids []string) error) error +} + +type OutpointIndex interface { + IndexBlockOutpoints(block *Block) error +} + +type AddressIndex interface { + IndexBlockAddresses(block *Block, txids map[string][]string) error +} + +var ( + chain = flag.String("chain", "mainnet", "none | mainnet | regtest | testnet3 | simnet") + + rpcURL = flag.String("rpcurl", "http://localhost:8332", "url of bitcoin RPC service") + rpcUser = flag.String("rpcuser", "rpc", "rpc username") + rpcPass = flag.String("rpcpass", "rpc", "rpc password") + rpcTimeout = flag.Uint("rpctimeout", 25, "rpc timeout in seconds") + rpcCache = flag.Int("rpccache", 50000, "number to tx replies to cache") + + dbPath = flag.String("path", "./data", "path to address index directory") + + blockHeight = flag.Int("blockheight", -1, "height of the starting block") + blockUntil = flag.Int("blockuntil", -1, "height of the final block") + queryAddress = flag.String("address", "", "query contents of this address") +) + +func main() { + flag.Parse() + + timeout := time.Duration(*rpcTimeout) * time.Second + rpc := NewBitcoinRPC(*rpcURL, *rpcUser, *rpcPass, timeout) + if *rpcCache > 0 { + rpc.EnableCache(*rpcCache) + } + + if *chain != "" { + for _, p := range GetChainParams() { + if p.Name == *chain { + rpc.Parser = &BitcoinBlockParser{Params: p} + } + } + if rpc.Parser == nil { + log.Fatal("unknown chain") + } + } + + db, err := NewRocksDB(*dbPath) + if err != nil { + log.Fatal(err) + } + defer db.Close() + + if *blockHeight >= 0 { + if *blockUntil < 0 { + *blockUntil = *blockHeight + } + height := uint32(*blockHeight) + until := uint32(*blockUntil) + address := *queryAddress + + if address != "" { + if err = db.GetAddressTransactions(address, height, until, printResult); err != nil { + log.Fatal(err) + } + } else { + if err = indexBlocks(rpc, rpc, db, db, height, until); err != nil { + log.Fatal(err) + } + } + } +} + +func printResult(txids []string) error { + for i, txid := range txids { + log.Printf("%d: %s", i, txid) + } + return nil +} + +func (b *Block) CollectBlockAddresses(o OutpointAddressOracle) (map[string][]string, error) { + addrs := make(map[string][]string, 0) + + for _, tx := range b.Txs { + voutAddrs, err := tx.CollectAddresses(o) + if err != nil { + return nil, err + } + for _, addr := range voutAddrs { + addrs[addr] = append(addrs[addr], tx.Txid) + } + } + + return addrs, nil +} + +func (tx *Tx) CollectAddresses(o OutpointAddressOracle) ([]string, error) { + addrs := make([]string, 0) + seen := make(map[string]struct{}) + + for _, vout := range tx.Vout { + for _, addr := range vout.ScriptPubKey.Addresses { + if _, found := seen[addr]; !found { + addrs = append(addrs, addr) + seen[addr] = struct{}{} + } + } + } + + for _, vin := range tx.Vin { + if vin.Coinbase != "" { + continue + } + vinAddrs, err := o.GetOutpointAddresses(vin.Txid, vin.Vout) + if err != nil { + return nil, err + } + for _, addr := range vinAddrs { + if _, found := seen[addr]; !found { + addrs = append(addrs, addr) + seen[addr] = struct{}{} + } + } + } + + return addrs, nil +} + +func indexBlocks(bo BlockOracle, oao OutpointAddressOracle, ai AddressIndex, oi OutpointIndex, lower uint32, higher uint32) error { + for height := lower; height <= higher; height++ { + hash, err := bo.GetBlockHash(height) + if err != nil { + return err + } + block, err := bo.GetBlock(hash, height) + if err != nil { + return err + } + addrs, err := block.CollectBlockAddresses(oao) + if err != nil { + return err + } + if err := oi.IndexBlockOutpoints(block); err != nil { + return err + } + if err := ai.IndexBlockAddresses(block, addrs); err != nil { + return err + } + } + return nil +} diff --git a/jsonrpc.go b/jsonrpc.go new file mode 100644 index 00000000..3c966127 --- /dev/null +++ b/jsonrpc.go @@ -0,0 +1,75 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "sync/atomic" +) + +type cmd struct { + ID uint32 `json:"id"` + Method string `json:"method"` + Params interface{} `json:"params"` +} + +type cmdError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +func (e *cmdError) Error() string { + return fmt.Sprintf("%d: %s", e.Code, e.Message) +} + +type cmdResult struct { + ID uint32 `json:"id"` + Result json.RawMessage `json:"result"` + Error *cmdError `json:"error"` +} + +// JSONRPC is a simple JSON-RPC HTTP client. +type JSONRPC struct { + counter uint32 + Client http.Client + URL string + User string + Password string +} + +// Call constructs a JSON-RPC request, sends it over HTTP client, and unmarshals +// the response result. +func (c *JSONRPC) Call(method string, result interface{}, params ...interface{}) error { + b, err := json.Marshal(&cmd{ + ID: c.nextID(), + Method: method, + Params: params, + }) + if err != nil { + return err + } + req, err := http.NewRequest("POST", c.URL, bytes.NewBuffer(b)) + if err != nil { + return err + } + req.SetBasicAuth(c.User, c.Password) + res, err := c.Client.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + d := json.NewDecoder(res.Body) + r := cmdResult{} + if err = d.Decode(&r); err != nil { + return err + } + if r.Error != nil { + return r.Error + } + return json.Unmarshal(r.Result, result) +} + +func (c *JSONRPC) nextID() uint32 { + return atomic.AddUint32(&c.counter, 1) +} diff --git a/rocksdb.go b/rocksdb.go new file mode 100644 index 00000000..bf3b8cf4 --- /dev/null +++ b/rocksdb.go @@ -0,0 +1,273 @@ +package main + +import ( + "bytes" + "encoding/binary" + "encoding/hex" + "errors" + "log" + + "github.com/bsm/go-vlq" + "github.com/btcsuite/btcutil/base58" + + "github.com/tecbot/gorocksdb" +) + +type RocksDB struct { + db *gorocksdb.DB + wo *gorocksdb.WriteOptions + ro *gorocksdb.ReadOptions +} + +// NewRocksDB opens an internal handle to RocksDB environment. Close +// needs to be called to release it. +func NewRocksDB(path string) (d *RocksDB, err error) { + log.Printf("rocksdb: open %s", path) + + fp := gorocksdb.NewBloomFilter(10) + bbto := gorocksdb.NewDefaultBlockBasedTableOptions() + bbto.SetBlockCache(gorocksdb.NewLRUCache(3 << 30)) + bbto.SetFilterPolicy(fp) + + opts := gorocksdb.NewDefaultOptions() + opts.SetBlockBasedTableFactory(bbto) + opts.SetCreateIfMissing(true) + opts.SetMaxBackgroundCompactions(4) + + db, err := gorocksdb.OpenDb(opts, path) + if err != nil { + return + } + + wo := gorocksdb.NewDefaultWriteOptions() + ro := gorocksdb.NewDefaultReadOptions() + ro.SetFillCache(false) + + return &RocksDB{db, wo, ro}, nil +} + +// Close releases the RocksDB environment opened in NewRocksDB. +func (d *RocksDB) Close() error { + log.Printf("rocksdb: close") + d.wo.Destroy() + d.ro.Destroy() + d.db.Close() + return nil +} + +func (d *RocksDB) GetAddressTransactions(address string, lower uint32, higher uint32, fn func(txids []string) error) (err error) { + log.Printf("rocksdb: address get %d:%d %s", lower, higher, address) + + kstart, err := packAddressKey(lower, address) + if err != nil { + return err + } + kstop, err := packAddressKey(higher, address) + if err != nil { + return err + } + + it := d.db.NewIterator(d.ro) + defer it.Close() + + for it.Seek(kstart); it.Valid(); it.Next() { + k := it.Key() + v := it.Value() + if bytes.Compare(k.Data(), kstop) > 0 { + break + } + txids, err := unpackAddressVal(v.Data()) + if err != nil { + return err + } + if err := fn(txids); err != nil { + return err + } + } + return nil +} + +// Address Index + +func (d *RocksDB) IndexBlockAddresses(block *Block, txids map[string][]string) error { + log.Printf("rocksdb: address put %d in %d %s", len(txids), block.Height, block.Hash) + + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + + for addr, txids := range txids { + k, err := packAddressKey(block.Height, addr) + if err != nil { + return err + } + v, err := packAddressVal(txids) + if err != nil { + return err + } + wb.Put(k, v) + } + + return d.db.Write(d.wo, wb) +} + +func packAddressKey(height uint32, address string) (b []byte, err error) { + b, err = packAddress(address) + if err != nil { + return + } + h := packUint(height) + b = append(b, h...) + return +} + +func packAddressVal(txids []string) (b []byte, err error) { + for _, txid := range txids { + t, err := packTxid(txid) + if err != nil { + return nil, err + } + b = append(b, t...) + } + return +} + +const transactionIDLen = 32 + +func unpackAddressVal(b []byte) (txids []string, err error) { + for i := 0; i < len(b); i += transactionIDLen { + t, err := unpackTxid(b[i : i+transactionIDLen]) + if err != nil { + return nil, err + } + txids = append(txids, t) + } + return +} + +// Outpoint index + +func (d *RocksDB) IndexBlockOutpoints(block *Block) error { + log.Printf("rocksdb: outpoints put %d in %d %s", len(block.Txs), block.Height, block.Hash) + + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + + bv, err := packBlockValue(block.Hash) + if err != nil { + return err + } + bk := packUint(block.Height) + wb.Put(bk, bv) + + for _, tx := range block.Txs { + for _, vout := range tx.Vout { + k, err := packOutpointKey(block.Height, tx.Txid, vout.N) + if err != nil { + return err + } + v, err := packOutpointValue(vout.ScriptPubKey.Addresses) + if err != nil { + return err + } + wb.Put(k, v) + } + } + + return d.db.Write(d.wo, wb) +} + +func packOutpointKey(height uint32, txid string, vout uint32) (b []byte, err error) { + h := packUint(height) + t, err := packTxid(txid) + if err != nil { + return nil, err + } + v := packVarint(vout) + b = append(b, h...) + b = append(b, t...) + b = append(b, v...) + return +} + +func packOutpointValue(addrs []string) (b []byte, err error) { + for _, addr := range addrs { + a, err := packAddress(addr) + if err != nil { + return nil, err + } + i := packVarint(uint32(len(a))) + b = append(b, i...) + b = append(b, a...) + } + return +} + +func unpackOutpointValue(b []byte) (addrs []string, err error) { + r := bytes.NewReader(b) + for r.Len() > 0 { + alen, err := vlq.ReadUint(r) + if err != nil { + return nil, err + } + abuf := make([]byte, alen) + _, err = r.Read(abuf) + if err != nil { + return nil, err + } + addr, err := unpackAddress(abuf) + if err != nil { + return nil, err + } + addrs = append(addrs, addr) + } + return +} + +// Helpers + +func packUint(i uint32) []byte { + b := make([]byte, 4) + binary.BigEndian.PutUint32(b, i) + return b +} + +func packVarint(i uint32) []byte { + b := make([]byte, vlq.MaxLen32) + n := vlq.PutUint(b, uint64(i)) + return b[:n] +} + +var ( + ErrInvalidAddress = errors.New("invalid address") +) + +func packAddress(s string) (b []byte, err error) { + b = base58.Decode(s) + if len(b) > 4 { + b = b[:len(b)-4] + } else { + err = ErrInvalidAddress + } + return +} + +func unpackAddress(b []byte) (s string, err error) { + if len(b) > 1 { + s = base58.CheckEncode(b[1:], b[0]) + } else { + err = ErrInvalidAddress + } + return +} + +func packTxid(s string) (b []byte, err error) { + return hex.DecodeString(s) +} + +func unpackTxid(b []byte) (s string, err error) { + return hex.EncodeToString(b), nil +} + +func packBlockValue(hash string) ([]byte, error) { + return hex.DecodeString(hash) +} diff --git a/types.go b/types.go new file mode 100644 index 00000000..8b1be566 --- /dev/null +++ b/types.go @@ -0,0 +1,46 @@ +package main + +type ScriptSig struct { + Asm string `json:"asm"` + Hex string `json:"hex"` +} + +type Vin struct { + Coinbase string `json:"coinbase"` + Txid string `json:"txid"` + Vout uint32 `json:"vout"` + ScriptSig ScriptSig `json:"scriptSig"` + Sequence uint32 `json:"sequence"` +} + +type ScriptPubKey struct { + Asm string `json:"asm"` + Hex string `json:"hex,omitempty"` + Type string `json:"type"` + Addresses []string `json:"addresses,omitempty"` +} + +type Vout struct { + Value float64 `json:"value"` + N uint32 `json:"n"` + ScriptPubKey ScriptPubKey `json:"scriptPubKey"` +} + +type Tx struct { + Txid string `json:"txid"` + Version int32 `json:"version"` + LockTime uint32 `json:"locktime"` + Vin []Vin `json:"vin"` + Vout []Vout `json:"vout"` + BlockHash string `json:"blockhash,omitempty"` + Confirmations uint32 `json:"confirmations,omitempty"` + Time int64 `json:"time,omitempty"` + Blocktime int64 `json:"blocktime,omitempty"` +} + +type Block struct { + Hash string `json:"hash"` + Height uint32 `json:"height"` + Txids []string `json:"tx"` + Txs []*Tx `json:"_"` +}