spacecruft
/
blockbook
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

initial commit

pull/1/head
Jan Pochyla 2017-08-28 17:50:57 +02:00
commit 9ada001e3c
9 changed files with 890 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored 100644
View File

@ -0,0 +1,3 @@
vendor/
blockbook
notes.txt

51
Gopkg.lock generated 100644
View File

@ -0,0 +1,51 @@
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
branch = "master"
name = "github.com/bsm/go-vlq"
packages = ["."]
revision = "ec6e8d4f5f4ec0f6e808ffc7f4dcc7516d4d7d49"
[[projects]]
branch = "master"
name = "github.com/btcsuite/btcd"
packages = ["btcec","chaincfg","chaincfg/chainhash","txscript","wire"]
revision = "a1d1ea70dd212a440beb9caa4b766a58d1ed0254"
[[projects]]
branch = "master"
name = "github.com/btcsuite/btclog"
packages = ["."]
revision = "84c8d2346e9fc8c7b947e243b9c24e6df9fd206a"
[[projects]]
branch = "master"
name = "github.com/btcsuite/btcutil"
packages = [".","base58","bech32"]
revision = "501929d3d046174c3d39f0ea54ece471aa17238c"
[[projects]]
branch = "master"
name = "github.com/hashicorp/golang-lru"
packages = [".","simplelru"]
revision = "0a025b7e63adc15a622f29b0b2c4c3848243bbf6"
[[projects]]
branch = "master"
name = "github.com/tecbot/gorocksdb"
packages = ["."]
revision = "b9cb0d30eca790f5bcf524dd3beb715d6c8d1923"
[[projects]]
branch = "master"
name = "golang.org/x/crypto"
packages = ["ripemd160"]
revision = "81e90905daefcd6fd217b62423c0908922eadb30"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "3bb30a1d65e170c9b0f282efa56af8096dc74f149744743dddd27f26e1880612"
solver-name = "gps-cdcl"
solver-version = 1

42
Gopkg.toml 100644
View File

@ -0,0 +1,42 @@
# Gopkg.toml example
#
# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
# for detailed Gopkg.toml documentation.
#
# required = ["github.com/user/thing/cmd/thing"]
# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
#
# [[constraint]]
# name = "github.com/user/project"
# version = "1.0.0"
#
# [[constraint]]
# name = "github.com/user/project2"
# branch = "dev"
# source = "github.com/myfork/project2"
#
# [[override]]
# name = "github.com/x/y"
# version = "2.4.0"
[[constraint]]
branch = "master"
name = "github.com/bsm/go-vlq"
[[constraint]]
branch = "master"
name = "github.com/btcsuite/btcd"
[[constraint]]
branch = "master"
name = "github.com/btcsuite/btcutil"
[[constraint]]
branch = "master"
name = "github.com/hashicorp/golang-lru"
[[constraint]]
branch = "master"
name = "github.com/tecbot/gorocksdb"

139
bitcoinrpc.go 100644
View File

@ -0,0 +1,139 @@
package main
import (
"encoding/hex"
"log"
"net/http"
"time"
lru "github.com/hashicorp/golang-lru"
)
// BitcoinRPC is an interface to JSON-RPC bitcoind service.
type BitcoinRPC struct {
client JSONRPC
Parser BlockParser
txCache *lru.Cache
}
// NewBitcoinRPC returns new BitcoinRPC instance.
func NewBitcoinRPC(url string, user string, password string, timeout time.Duration) *BitcoinRPC {
return &BitcoinRPC{
client: JSONRPC{
Client: http.Client{Timeout: timeout},
URL: url,
User: user,
Password: password,
},
}
}
// EnableCache turns on LRU caching for transaction lookups.
func (b *BitcoinRPC) EnableCache(size int) error {
c, err := lru.New(size)
if err != nil {
return err
}
b.txCache = c
return nil
}
// ClearCache purges the cache used for transaction results.
func (b *BitcoinRPC) ClearCache() {
if b.txCache != nil {
b.txCache.Purge()
}
}
// GetBlock returns information about the block with the given hash.
func (b *BitcoinRPC) GetBlock(hash string, height uint32) (block *Block, err error) {
if b.Parser != nil {
return b.GetBlockAndParse(hash, height)
} else {
return b.GetParsedBlock(hash)
}
}
// GetBlockAndParse returns information about the block with the given hash.
//
// It downloads raw block and parses it in-process.
func (b *BitcoinRPC) GetBlockAndParse(hash string, height uint32) (block *Block, err error) {
log.Printf("rpc: getblock (verbose=false) %v", hash)
var raw string
err = b.client.Call("getblock", &raw, hash, false) // verbose=false
if err != nil {
return
}
data, err := hex.DecodeString(raw)
if err != nil {
return
}
block, err = b.Parser.ParseBlock(data)
if err == nil {
block.Hash = hash
block.Height = height
}
return
}
// GetParsedBlock returns information about the block with the given hash.
//
// It downloads parsed block with transaction IDs and then looks them up,
// one by one.
func (b *BitcoinRPC) GetParsedBlock(hash string) (block *Block, err error) {
log.Printf("rpc: getblock (verbose=true) %v", hash)
block = &Block{}
err = b.client.Call("getblock", block, hash, true) // verbose=true
if err != nil {
return
}
for _, txid := range block.Txids {
tx, err := b.GetTransaction(txid)
if err != nil {
return nil, err
}
block.Txs = append(block.Txs, tx)
}
return
}
// GetBlockHash returns hash of block in best-block-chain at given height.
func (b *BitcoinRPC) GetBlockHash(height uint32) (hash string, err error) {
log.Printf("rpc: getblockhash %v", height)
err = b.client.Call("getblockhash", &hash, height)
return
}
// GetBlockCount returns the number of blocks in the longest chain.
func (b *BitcoinRPC) GetBlockCount() (count uint32, err error) {
log.Printf("rpc: getblockcount")
err = b.client.Call("getblockcount", &count)
return
}
// GetRawTransaction returns the number of blocks in the longest chain. If the
// transaction cache is turned on, returned RawTx.Confirmations is stale.
func (b *BitcoinRPC) GetTransaction(txid string) (tx *Tx, err error) {
if b.txCache != nil {
if cachedTx, ok := b.txCache.Get(txid); ok {
tx = cachedTx.(*Tx)
return
}
}
log.Printf("rpc: getrawtransaction %v", txid)
tx = &Tx{}
err = b.client.Call("getrawtransaction", tx, txid, true) // verbose = true
if b.txCache != nil {
b.txCache.Add(txid, tx)
}
return
}
// GetOutpointAddresses returns all unique addresses from given transaction output.
func (b *BitcoinRPC) GetOutpointAddresses(txid string, vout uint32) ([]string, error) {
tx, err := b.GetTransaction(txid)
if err != nil {
return nil, err
}
return tx.Vout[vout].ScriptPubKey.Addresses, nil
}

88
bitcoinwire.go 100644
View File

@ -0,0 +1,88 @@
package main
import (
"bytes"
"encoding/hex"
"github.com/btcsuite/btcd/chaincfg"
"github.com/btcsuite/btcd/txscript"
"github.com/btcsuite/btcd/wire"
)
func GetChainParams() []*chaincfg.Params {
return []*chaincfg.Params{
&chaincfg.MainNetParams,
&chaincfg.RegressionNetParams,
&chaincfg.TestNet3Params,
&chaincfg.SimNetParams,
}
}
type BitcoinBlockParser struct {
Params *chaincfg.Params
}
func (p *BitcoinBlockParser) parseOutputScript(b []byte) (addrs []string, err error) {
_, addresses, _, err := txscript.ExtractPkScriptAddrs(b, p.Params)
for _, a := range addresses {
addr := a.EncodeAddress()
addrs = append(addrs, addr)
}
return addrs, err
}
func (p *BitcoinBlockParser) ParseBlock(b []byte) (*Block, error) {
w := wire.MsgBlock{}
r := bytes.NewReader(b)
if err := w.DeserializeNoWitness(r); err != nil {
return nil, err
}
block := &Block{}
for _, t := range w.Transactions {
tx := &Tx{
Txid: t.TxHash().String(),
Version: t.Version,
LockTime: t.LockTime,
Vin: make([]Vin, len(t.TxIn)),
Vout: make([]Vout, len(t.TxOut)),
// missing: BlockHash,
// missing: Confirmations,
// missing: Time,
// missing: Blocktime,
}
for i, in := range t.TxIn {
s := ScriptSig{
Hex: hex.EncodeToString(in.SignatureScript),
// missing: Asm,
}
tx.Vin[i] = Vin{
Coinbase: "_",
Txid: in.PreviousOutPoint.Hash.String(),
Vout: in.PreviousOutPoint.Index,
Sequence: in.Sequence,
ScriptSig: s,
}
}
for i, out := range t.TxOut {
addrs, err := p.parseOutputScript(out.PkScript)
if err != nil {
return nil, err
}
s := ScriptPubKey{
Hex: hex.EncodeToString(out.PkScript),
Addresses: addrs,
// missing: Asm,
// missing: Type,
}
tx.Vout[i] = Vout{
Value: float64(out.Value),
N: uint32(i),
ScriptPubKey: s,
}
}
block.Txs = append(block.Txs, tx)
}
return block, nil
}

173
blockbook.go 100644
View File

@ -0,0 +1,173 @@
package main
import (
"flag"
"log"
"time"
)
type BlockParser interface {
ParseBlock(b []byte) (*Block, error)
}
type BlockOracle interface {
GetBlockHash(height uint32) (string, error)
GetBlock(hash string, height uint32) (*Block, error)
}
type OutpointAddressOracle interface {
GetOutpointAddresses(txid string, vout uint32) ([]string, error)
}
type AddressTransactionOracle interface {
GetAddressTransactions(address string, lower uint32, higher uint32, fn func(txids []string) error) error
}
type OutpointIndex interface {
IndexBlockOutpoints(block *Block) error
}
type AddressIndex interface {
IndexBlockAddresses(block *Block, txids map[string][]string) error
}
var (
chain = flag.String("chain", "mainnet", "none | mainnet | regtest | testnet3 | simnet")
rpcURL = flag.String("rpcurl", "http://localhost:8332", "url of bitcoin RPC service")
rpcUser = flag.String("rpcuser", "rpc", "rpc username")
rpcPass = flag.String("rpcpass", "rpc", "rpc password")
rpcTimeout = flag.Uint("rpctimeout", 25, "rpc timeout in seconds")
rpcCache = flag.Int("rpccache", 50000, "number to tx replies to cache")
dbPath = flag.String("path", "./data", "path to address index directory")
blockHeight = flag.Int("blockheight", -1, "height of the starting block")
blockUntil = flag.Int("blockuntil", -1, "height of the final block")
queryAddress = flag.String("address", "", "query contents of this address")
)
func main() {
flag.Parse()
timeout := time.Duration(*rpcTimeout) * time.Second
rpc := NewBitcoinRPC(*rpcURL, *rpcUser, *rpcPass, timeout)
if *rpcCache > 0 {
rpc.EnableCache(*rpcCache)
}
if *chain != "" {
for _, p := range GetChainParams() {
if p.Name == *chain {
rpc.Parser = &BitcoinBlockParser{Params: p}
}
}
if rpc.Parser == nil {
log.Fatal("unknown chain")
}
}
db, err := NewRocksDB(*dbPath)
if err != nil {
log.Fatal(err)
}
defer db.Close()
if *blockHeight >= 0 {
if *blockUntil < 0 {
*blockUntil = *blockHeight
}
height := uint32(*blockHeight)
until := uint32(*blockUntil)
address := *queryAddress
if address != "" {
if err = db.GetAddressTransactions(address, height, until, printResult); err != nil {
log.Fatal(err)
}
} else {
if err = indexBlocks(rpc, rpc, db, db, height, until); err != nil {
log.Fatal(err)
}
}
}
}
func printResult(txids []string) error {
for i, txid := range txids {
log.Printf("%d: %s", i, txid)
}
return nil
}
func (b *Block) CollectBlockAddresses(o OutpointAddressOracle) (map[string][]string, error) {
addrs := make(map[string][]string, 0)
for _, tx := range b.Txs {
voutAddrs, err := tx.CollectAddresses(o)
if err != nil {
return nil, err
}
for _, addr := range voutAddrs {
addrs[addr] = append(addrs[addr], tx.Txid)
}
}
return addrs, nil
}
func (tx *Tx) CollectAddresses(o OutpointAddressOracle) ([]string, error) {
addrs := make([]string, 0)
seen := make(map[string]struct{})
for _, vout := range tx.Vout {
for _, addr := range vout.ScriptPubKey.Addresses {
if _, found := seen[addr]; !found {
addrs = append(addrs, addr)
seen[addr] = struct{}{}
}
}
}
for _, vin := range tx.Vin {
if vin.Coinbase != "" {
continue
}
vinAddrs, err := o.GetOutpointAddresses(vin.Txid, vin.Vout)
if err != nil {
return nil, err
}
for _, addr := range vinAddrs {
if _, found := seen[addr]; !found {
addrs = append(addrs, addr)
seen[addr] = struct{}{}
}
}
}
return addrs, nil
}
func indexBlocks(bo BlockOracle, oao OutpointAddressOracle, ai AddressIndex, oi OutpointIndex, lower uint32, higher uint32) error {
for height := lower; height <= higher; height++ {
hash, err := bo.GetBlockHash(height)
if err != nil {
return err
}
block, err := bo.GetBlock(hash, height)
if err != nil {
return err
}
addrs, err := block.CollectBlockAddresses(oao)
if err != nil {
return err
}
if err := oi.IndexBlockOutpoints(block); err != nil {
return err
}
if err := ai.IndexBlockAddresses(block, addrs); err != nil {
return err
}
}
return nil
}

75
jsonrpc.go 100644
View File

@ -0,0 +1,75 @@
package main
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"sync/atomic"
)
type cmd struct {
ID uint32 `json:"id"`
Method string `json:"method"`
Params interface{} `json:"params"`
}
type cmdError struct {
Code int `json:"code"`
Message string `json:"message"`
}
func (e *cmdError) Error() string {
return fmt.Sprintf("%d: %s", e.Code, e.Message)
}
type cmdResult struct {
ID uint32 `json:"id"`
Result json.RawMessage `json:"result"`
Error *cmdError `json:"error"`
}
// JSONRPC is a simple JSON-RPC HTTP client.
type JSONRPC struct {
counter uint32
Client http.Client
URL string
User string
Password string
}
// Call constructs a JSON-RPC request, sends it over HTTP client, and unmarshals
// the response result.
func (c *JSONRPC) Call(method string, result interface{}, params ...interface{}) error {
b, err := json.Marshal(&cmd{
ID: c.nextID(),
Method: method,
Params: params,
})
if err != nil {
return err
}
req, err := http.NewRequest("POST", c.URL, bytes.NewBuffer(b))
if err != nil {
return err
}
req.SetBasicAuth(c.User, c.Password)
res, err := c.Client.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
d := json.NewDecoder(res.Body)
r := cmdResult{}
if err = d.Decode(&r); err != nil {
return err
}
if r.Error != nil {
return r.Error
}
return json.Unmarshal(r.Result, result)
}
func (c *JSONRPC) nextID() uint32 {
return atomic.AddUint32(&c.counter, 1)
}

273
rocksdb.go 100644
View File

@ -0,0 +1,273 @@
package main
import (
"bytes"
"encoding/binary"
"encoding/hex"
"errors"
"log"
"github.com/bsm/go-vlq"
"github.com/btcsuite/btcutil/base58"
"github.com/tecbot/gorocksdb"
)
type RocksDB struct {
db *gorocksdb.DB
wo *gorocksdb.WriteOptions
ro *gorocksdb.ReadOptions
}
// NewRocksDB opens an internal handle to RocksDB environment. Close
// needs to be called to release it.
func NewRocksDB(path string) (d *RocksDB, err error) {
log.Printf("rocksdb: open %s", path)
fp := gorocksdb.NewBloomFilter(10)
bbto := gorocksdb.NewDefaultBlockBasedTableOptions()
bbto.SetBlockCache(gorocksdb.NewLRUCache(3 << 30))
bbto.SetFilterPolicy(fp)
opts := gorocksdb.NewDefaultOptions()
opts.SetBlockBasedTableFactory(bbto)
opts.SetCreateIfMissing(true)
opts.SetMaxBackgroundCompactions(4)
db, err := gorocksdb.OpenDb(opts, path)
if err != nil {
return
}
wo := gorocksdb.NewDefaultWriteOptions()
ro := gorocksdb.NewDefaultReadOptions()
ro.SetFillCache(false)
return &RocksDB{db, wo, ro}, nil
}
// Close releases the RocksDB environment opened in NewRocksDB.
func (d *RocksDB) Close() error {
log.Printf("rocksdb: close")
d.wo.Destroy()
d.ro.Destroy()
d.db.Close()
return nil
}
func (d *RocksDB) GetAddressTransactions(address string, lower uint32, higher uint32, fn func(txids []string) error) (err error) {
log.Printf("rocksdb: address get %d:%d %s", lower, higher, address)
kstart, err := packAddressKey(lower, address)
if err != nil {
return err
}
kstop, err := packAddressKey(higher, address)
if err != nil {
return err
}
it := d.db.NewIterator(d.ro)
defer it.Close()
for it.Seek(kstart); it.Valid(); it.Next() {
k := it.Key()
v := it.Value()
if bytes.Compare(k.Data(), kstop) > 0 {
break
}
txids, err := unpackAddressVal(v.Data())
if err != nil {
return err
}
if err := fn(txids); err != nil {
return err
}
}
return nil
}
// Address Index
func (d *RocksDB) IndexBlockAddresses(block *Block, txids map[string][]string) error {
log.Printf("rocksdb: address put %d in %d %s", len(txids), block.Height, block.Hash)
wb := gorocksdb.NewWriteBatch()
defer wb.Destroy()
for addr, txids := range txids {
k, err := packAddressKey(block.Height, addr)
if err != nil {
return err
}
v, err := packAddressVal(txids)
if err != nil {
return err
}
wb.Put(k, v)
}
return d.db.Write(d.wo, wb)
}
func packAddressKey(height uint32, address string) (b []byte, err error) {
b, err = packAddress(address)
if err != nil {
return
}
h := packUint(height)
b = append(b, h...)
return
}
func packAddressVal(txids []string) (b []byte, err error) {
for _, txid := range txids {
t, err := packTxid(txid)
if err != nil {
return nil, err
}
b = append(b, t...)
}
return
}
const transactionIDLen = 32
func unpackAddressVal(b []byte) (txids []string, err error) {
for i := 0; i < len(b); i += transactionIDLen {
t, err := unpackTxid(b[i : i+transactionIDLen])
if err != nil {
return nil, err
}
txids = append(txids, t)
}
return
}
// Outpoint index
func (d *RocksDB) IndexBlockOutpoints(block *Block) error {
log.Printf("rocksdb: outpoints put %d in %d %s", len(block.Txs), block.Height, block.Hash)
wb := gorocksdb.NewWriteBatch()
defer wb.Destroy()
bv, err := packBlockValue(block.Hash)
if err != nil {
return err
}
bk := packUint(block.Height)
wb.Put(bk, bv)
for _, tx := range block.Txs {
for _, vout := range tx.Vout {
k, err := packOutpointKey(block.Height, tx.Txid, vout.N)
if err != nil {
return err
}
v, err := packOutpointValue(vout.ScriptPubKey.Addresses)
if err != nil {
return err
}
wb.Put(k, v)
}
}
return d.db.Write(d.wo, wb)
}
func packOutpointKey(height uint32, txid string, vout uint32) (b []byte, err error) {
h := packUint(height)
t, err := packTxid(txid)
if err != nil {
return nil, err
}
v := packVarint(vout)
b = append(b, h...)
b = append(b, t...)
b = append(b, v...)
return
}
func packOutpointValue(addrs []string) (b []byte, err error) {
for _, addr := range addrs {
a, err := packAddress(addr)
if err != nil {
return nil, err
}
i := packVarint(uint32(len(a)))
b = append(b, i...)
b = append(b, a...)
}
return
}
func unpackOutpointValue(b []byte) (addrs []string, err error) {
r := bytes.NewReader(b)
for r.Len() > 0 {
alen, err := vlq.ReadUint(r)
if err != nil {
return nil, err
}
abuf := make([]byte, alen)
_, err = r.Read(abuf)
if err != nil {
return nil, err
}
addr, err := unpackAddress(abuf)
if err != nil {
return nil, err
}
addrs = append(addrs, addr)
}
return
}
// Helpers
func packUint(i uint32) []byte {
b := make([]byte, 4)
binary.BigEndian.PutUint32(b, i)
return b
}
func packVarint(i uint32) []byte {
b := make([]byte, vlq.MaxLen32)
n := vlq.PutUint(b, uint64(i))
return b[:n]
}
var (
ErrInvalidAddress = errors.New("invalid address")
)
func packAddress(s string) (b []byte, err error) {
b = base58.Decode(s)
if len(b) > 4 {
b = b[:len(b)-4]
} else {
err = ErrInvalidAddress
}
return
}
func unpackAddress(b []byte) (s string, err error) {
if len(b) > 1 {
s = base58.CheckEncode(b[1:], b[0])
} else {
err = ErrInvalidAddress
}
return
}
func packTxid(s string) (b []byte, err error) {
return hex.DecodeString(s)
}
func unpackTxid(b []byte) (s string, err error) {
return hex.EncodeToString(b), nil
}
func packBlockValue(hash string) ([]byte, error) {
return hex.DecodeString(hash)
}

46
types.go 100644
View File

@ -0,0 +1,46 @@
package main
type ScriptSig struct {
Asm string `json:"asm"`
Hex string `json:"hex"`
}
type Vin struct {
Coinbase string `json:"coinbase"`
Txid string `json:"txid"`
Vout uint32 `json:"vout"`
ScriptSig ScriptSig `json:"scriptSig"`
Sequence uint32 `json:"sequence"`
}
type ScriptPubKey struct {
Asm string `json:"asm"`
Hex string `json:"hex,omitempty"`
Type string `json:"type"`
Addresses []string `json:"addresses,omitempty"`
}
type Vout struct {
Value float64 `json:"value"`
N uint32 `json:"n"`
ScriptPubKey ScriptPubKey `json:"scriptPubKey"`
}
type Tx struct {
Txid string `json:"txid"`
Version int32 `json:"version"`
LockTime uint32 `json:"locktime"`
Vin []Vin `json:"vin"`
Vout []Vout `json:"vout"`
BlockHash string `json:"blockhash,omitempty"`
Confirmations uint32 `json:"confirmations,omitempty"`
Time int64 `json:"time,omitempty"`
Blocktime int64 `json:"blocktime,omitempty"`
}
type Block struct {
Hash string `json:"hash"`
Height uint32 `json:"height"`
Txids []string `json:"tx"`
Txs []*Tx `json:"_"`
}