2017-08-28 09:50:57 -06:00
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
|
|
|
"flag"
|
|
|
|
"log"
|
2017-10-05 06:35:07 -06:00
|
|
|
"sync"
|
2017-08-28 09:50:57 -06:00
|
|
|
"time"
|
2017-09-12 18:50:34 -06:00
|
|
|
|
2018-01-18 08:44:31 -07:00
|
|
|
"blockbook/db"
|
|
|
|
"blockbook/server"
|
|
|
|
|
2017-09-12 18:50:34 -06:00
|
|
|
"github.com/pkg/profile"
|
2017-08-28 09:50:57 -06:00
|
|
|
)
|
|
|
|
|
2017-10-05 06:35:07 -06:00
|
|
|
type Blockchain interface {
|
2017-09-06 02:59:40 -06:00
|
|
|
GetBestBlockHash() (string, error)
|
2017-08-28 09:50:57 -06:00
|
|
|
GetBlockHash(height uint32) (string, error)
|
2017-09-11 04:20:21 -06:00
|
|
|
GetBlockHeader(hash string) (*BlockHeader, error)
|
2017-09-06 02:59:40 -06:00
|
|
|
GetBlock(hash string) (*Block, error)
|
2017-08-28 09:50:57 -06:00
|
|
|
}
|
|
|
|
|
2017-10-05 06:35:07 -06:00
|
|
|
type Index interface {
|
|
|
|
GetBestBlockHash() (string, error)
|
|
|
|
GetBlockHash(height uint32) (string, error)
|
|
|
|
GetTransactions(address string, lower uint32, higher uint32, fn func(txid string) error) error
|
|
|
|
ConnectBlock(block *Block) error
|
|
|
|
DisconnectBlock(block *Block) error
|
2017-08-28 09:50:57 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
var (
|
|
|
|
rpcURL = flag.String("rpcurl", "http://localhost:8332", "url of bitcoin RPC service")
|
|
|
|
rpcUser = flag.String("rpcuser", "rpc", "rpc username")
|
|
|
|
rpcPass = flag.String("rpcpass", "rpc", "rpc password")
|
|
|
|
rpcTimeout = flag.Uint("rpctimeout", 25, "rpc timeout in seconds")
|
|
|
|
|
|
|
|
dbPath = flag.String("path", "./data", "path to address index directory")
|
|
|
|
|
2017-10-05 06:35:07 -06:00
|
|
|
blockHeight = flag.Int("blockheight", -1, "height of the starting block")
|
|
|
|
blockUntil = flag.Int("blockuntil", -1, "height of the final block")
|
|
|
|
|
2017-08-28 09:50:57 -06:00
|
|
|
queryAddress = flag.String("address", "", "query contents of this address")
|
2017-09-11 04:20:21 -06:00
|
|
|
|
|
|
|
resync = flag.Bool("resync", false, "resync until tip")
|
2017-09-12 08:53:40 -06:00
|
|
|
repair = flag.Bool("repair", false, "repair the database")
|
2017-09-12 18:50:34 -06:00
|
|
|
prof = flag.Bool("prof", false, "profile program execution")
|
2017-10-06 04:57:51 -06:00
|
|
|
|
|
|
|
syncChunk = flag.Int("chunk", 100, "block chunk size for processing")
|
|
|
|
syncWorkers = flag.Int("workers", 8, "number of workers to process blocks")
|
|
|
|
dryRun = flag.Bool("dryrun", false, "do not index blocks, only download")
|
2017-10-07 03:05:35 -06:00
|
|
|
parse = flag.Bool("parse", false, "use in-process block parsing")
|
2018-01-18 08:44:31 -07:00
|
|
|
|
|
|
|
httpServer = flag.Bool("http", true, "run http server (default true)")
|
2017-08-28 09:50:57 -06:00
|
|
|
)
|
|
|
|
|
|
|
|
func main() {
|
|
|
|
flag.Parse()
|
|
|
|
|
2017-09-12 08:53:40 -06:00
|
|
|
if *repair {
|
|
|
|
if err := RepairRocksDB(*dbPath); err != nil {
|
|
|
|
log.Fatal(err)
|
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2017-10-05 06:35:07 -06:00
|
|
|
if *prof {
|
|
|
|
defer profile.Start().Stop()
|
|
|
|
}
|
|
|
|
|
|
|
|
rpc := NewBitcoinRPC(
|
|
|
|
*rpcURL,
|
|
|
|
*rpcUser,
|
|
|
|
*rpcPass,
|
|
|
|
time.Duration(*rpcTimeout)*time.Second)
|
2017-08-28 09:50:57 -06:00
|
|
|
|
2017-10-07 03:05:35 -06:00
|
|
|
if *parse {
|
|
|
|
rpc.Parser = &BitcoinBlockParser{
|
|
|
|
Params: GetChainParams()[0],
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-01-18 08:44:31 -07:00
|
|
|
db, err := db.NewRocksDB(*dbPath)
|
2017-08-28 09:50:57 -06:00
|
|
|
if err != nil {
|
|
|
|
log.Fatal(err)
|
|
|
|
}
|
|
|
|
defer db.Close()
|
|
|
|
|
2018-01-18 08:44:31 -07:00
|
|
|
if *httpServer {
|
|
|
|
s, err := server.New(db)
|
|
|
|
if err != nil {
|
|
|
|
log.Fatalf("https: %s", err)
|
|
|
|
}
|
|
|
|
err = s.Run()
|
|
|
|
if err != nil {
|
|
|
|
log.Fatalf("https: %s", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-09-12 16:36:08 -06:00
|
|
|
if *resync {
|
2017-10-05 06:35:07 -06:00
|
|
|
if err := resyncIndex(rpc, db); err != nil {
|
2017-09-12 16:36:08 -06:00
|
|
|
log.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-08-28 09:50:57 -06:00
|
|
|
if *blockHeight >= 0 {
|
|
|
|
if *blockUntil < 0 {
|
|
|
|
*blockUntil = *blockHeight
|
|
|
|
}
|
|
|
|
height := uint32(*blockHeight)
|
|
|
|
until := uint32(*blockUntil)
|
|
|
|
address := *queryAddress
|
|
|
|
|
|
|
|
if address != "" {
|
2017-09-06 07:36:55 -06:00
|
|
|
if err = db.GetTransactions(address, height, until, printResult); err != nil {
|
2017-08-28 09:50:57 -06:00
|
|
|
log.Fatal(err)
|
|
|
|
}
|
|
|
|
} else {
|
2017-10-06 04:57:51 -06:00
|
|
|
if err = connectBlocksParallel(
|
|
|
|
rpc,
|
|
|
|
db,
|
|
|
|
height,
|
|
|
|
until,
|
|
|
|
*syncChunk,
|
|
|
|
*syncWorkers,
|
|
|
|
); err != nil {
|
2017-08-28 09:50:57 -06:00
|
|
|
log.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-10-05 06:35:07 -06:00
|
|
|
func printResult(txid string) error {
|
|
|
|
log.Printf("%s", txid)
|
2017-08-28 09:50:57 -06:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2017-10-05 06:35:07 -06:00
|
|
|
func resyncIndex(chain Blockchain, index Index) error {
|
|
|
|
remote, err := chain.GetBestBlockHash()
|
2017-09-11 04:20:21 -06:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2017-10-05 06:35:07 -06:00
|
|
|
local, err := index.GetBestBlockHash()
|
2017-09-11 04:20:21 -06:00
|
|
|
if err != nil {
|
2017-10-05 06:35:07 -06:00
|
|
|
local = ""
|
2017-09-11 04:20:21 -06:00
|
|
|
}
|
2017-08-28 09:50:57 -06:00
|
|
|
|
2017-09-11 04:20:21 -06:00
|
|
|
// If the local block is missing, we're indexing from the genesis block.
|
2017-10-05 06:35:07 -06:00
|
|
|
if local == "" {
|
2017-09-11 04:20:21 -06:00
|
|
|
log.Printf("resync: genesis")
|
|
|
|
|
2017-10-05 06:35:07 -06:00
|
|
|
hash, err := chain.GetBlockHash(0)
|
2017-08-28 09:50:57 -06:00
|
|
|
if err != nil {
|
2017-09-11 04:20:21 -06:00
|
|
|
return err
|
2017-08-28 09:50:57 -06:00
|
|
|
}
|
2017-10-05 06:35:07 -06:00
|
|
|
return connectBlock(chain, index, hash)
|
2017-08-28 09:50:57 -06:00
|
|
|
}
|
|
|
|
|
2017-09-11 04:20:21 -06:00
|
|
|
// If the locally indexed block is the same as the best block on the
|
|
|
|
// network, we're done.
|
2017-10-05 06:35:07 -06:00
|
|
|
if local == remote {
|
|
|
|
log.Printf("resync: synced on %s", local)
|
2017-09-11 04:20:21 -06:00
|
|
|
return nil
|
|
|
|
}
|
2017-09-06 03:03:23 -06:00
|
|
|
|
2017-09-11 04:20:21 -06:00
|
|
|
// Is local tip on the best chain?
|
2017-10-05 06:35:07 -06:00
|
|
|
header, err := chain.GetBlockHeader(local)
|
2017-09-11 04:20:21 -06:00
|
|
|
forked := false
|
|
|
|
if err != nil {
|
2017-09-12 10:08:01 -06:00
|
|
|
if e, ok := err.(*RPCError); ok && e.Message == "Block not found" {
|
2017-09-11 04:20:21 -06:00
|
|
|
forked = true
|
|
|
|
} else {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
if header.Confirmations < 0 {
|
|
|
|
forked = true
|
2017-09-06 03:03:23 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-09-11 04:20:21 -06:00
|
|
|
if forked {
|
|
|
|
log.Printf("resync: local is forked")
|
|
|
|
// TODO: resync after disconnecting
|
2017-10-05 06:35:07 -06:00
|
|
|
return disconnectBlock(chain, index, header.Hash)
|
2017-09-11 04:20:21 -06:00
|
|
|
} else {
|
|
|
|
log.Printf("resync: local is behind")
|
2017-10-05 06:35:07 -06:00
|
|
|
return connectBlock(chain, index, header.Next)
|
2017-09-11 04:20:21 -06:00
|
|
|
}
|
|
|
|
}
|
2017-08-28 09:50:57 -06:00
|
|
|
|
2017-09-11 04:20:21 -06:00
|
|
|
func connectBlock(
|
2017-10-05 06:35:07 -06:00
|
|
|
chain Blockchain,
|
|
|
|
index Index,
|
2017-09-11 04:20:21 -06:00
|
|
|
hash string,
|
|
|
|
) error {
|
2017-09-11 07:06:16 -06:00
|
|
|
bch := make(chan blockResult, 8)
|
|
|
|
done := make(chan struct{})
|
|
|
|
defer close(done)
|
|
|
|
|
2017-10-05 06:35:07 -06:00
|
|
|
go getBlockChain(hash, chain, bch, done)
|
2017-09-11 07:06:16 -06:00
|
|
|
|
|
|
|
for res := range bch {
|
2017-10-05 06:35:07 -06:00
|
|
|
if res.err != nil {
|
|
|
|
return res.err
|
2017-08-28 09:50:57 -06:00
|
|
|
}
|
2017-10-05 06:35:07 -06:00
|
|
|
err := index.ConnectBlock(res.block)
|
2017-08-28 09:50:57 -06:00
|
|
|
if err != nil {
|
2017-09-11 04:20:21 -06:00
|
|
|
return err
|
2017-08-28 09:50:57 -06:00
|
|
|
}
|
2017-09-06 07:36:55 -06:00
|
|
|
}
|
2017-09-11 04:20:21 -06:00
|
|
|
|
|
|
|
return nil
|
2017-08-28 09:50:57 -06:00
|
|
|
}
|
|
|
|
|
2017-09-11 04:20:21 -06:00
|
|
|
func disconnectBlock(
|
2017-10-05 06:35:07 -06:00
|
|
|
chain Blockchain,
|
|
|
|
index Index,
|
2017-09-11 04:20:21 -06:00
|
|
|
hash string,
|
|
|
|
) error {
|
|
|
|
return nil
|
2017-09-06 07:36:55 -06:00
|
|
|
}
|
|
|
|
|
2017-10-05 06:35:07 -06:00
|
|
|
func connectBlocksParallel(
|
|
|
|
chain Blockchain,
|
|
|
|
index Index,
|
2017-09-04 06:16:37 -06:00
|
|
|
lower uint32,
|
|
|
|
higher uint32,
|
2017-10-06 04:57:51 -06:00
|
|
|
chunkSize int,
|
|
|
|
numWorkers int,
|
2017-09-04 06:16:37 -06:00
|
|
|
) error {
|
2017-10-05 06:35:07 -06:00
|
|
|
var wg sync.WaitGroup
|
2017-09-04 06:16:37 -06:00
|
|
|
|
2017-10-05 06:35:07 -06:00
|
|
|
work := func(i int) {
|
|
|
|
defer wg.Done()
|
|
|
|
|
|
|
|
offset := uint32(chunkSize * i)
|
|
|
|
stride := uint32(chunkSize * numWorkers)
|
|
|
|
|
2017-10-07 02:42:31 -06:00
|
|
|
for low := lower + offset; low <= higher; low += stride {
|
2017-10-06 04:57:51 -06:00
|
|
|
high := low + uint32(chunkSize-1)
|
2017-10-05 06:35:07 -06:00
|
|
|
if high > higher {
|
|
|
|
high = higher
|
|
|
|
}
|
|
|
|
err := connectBlockChunk(chain, index, low, high)
|
|
|
|
if err != nil {
|
|
|
|
log.Fatal(err) // TODO
|
|
|
|
}
|
2017-08-28 09:50:57 -06:00
|
|
|
}
|
2017-09-04 06:16:37 -06:00
|
|
|
}
|
2017-10-05 06:35:07 -06:00
|
|
|
for i := 0; i < numWorkers; i++ {
|
|
|
|
wg.Add(1)
|
|
|
|
go work(i)
|
|
|
|
}
|
|
|
|
wg.Wait()
|
2017-09-04 06:16:37 -06:00
|
|
|
|
2017-10-05 06:35:07 -06:00
|
|
|
return nil
|
2017-09-04 06:16:37 -06:00
|
|
|
}
|
|
|
|
|
2017-10-05 06:35:07 -06:00
|
|
|
func connectBlockChunk(
|
|
|
|
chain Blockchain,
|
|
|
|
index Index,
|
2017-09-11 04:20:21 -06:00
|
|
|
lower uint32,
|
|
|
|
higher uint32,
|
2017-10-05 06:35:07 -06:00
|
|
|
) error {
|
|
|
|
connected, err := isBlockConnected(chain, index, higher)
|
|
|
|
if err != nil || connected {
|
|
|
|
return err
|
|
|
|
}
|
2017-09-06 02:59:40 -06:00
|
|
|
|
|
|
|
height := lower
|
2017-10-05 06:35:07 -06:00
|
|
|
hash, err := chain.GetBlockHash(lower)
|
2017-09-06 03:03:23 -06:00
|
|
|
if err != nil {
|
2017-10-05 06:35:07 -06:00
|
|
|
return err
|
2017-09-06 03:03:23 -06:00
|
|
|
}
|
2017-09-06 02:59:40 -06:00
|
|
|
|
|
|
|
for height <= higher {
|
2017-10-05 06:35:07 -06:00
|
|
|
block, err := chain.GetBlock(hash)
|
2017-09-04 06:16:37 -06:00
|
|
|
if err != nil {
|
2017-10-05 06:35:07 -06:00
|
|
|
return err
|
2017-08-28 09:50:57 -06:00
|
|
|
}
|
2017-09-06 02:59:40 -06:00
|
|
|
hash = block.Next
|
|
|
|
height = block.Height + 1
|
2017-10-06 04:57:51 -06:00
|
|
|
if *dryRun {
|
|
|
|
continue
|
|
|
|
}
|
2017-10-05 06:35:07 -06:00
|
|
|
err = index.ConnectBlock(block)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2017-08-28 09:50:57 -06:00
|
|
|
}
|
2017-10-05 06:35:07 -06:00
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func isBlockConnected(
|
|
|
|
chain Blockchain,
|
|
|
|
index Index,
|
|
|
|
height uint32,
|
|
|
|
) (bool, error) {
|
|
|
|
local, err := index.GetBlockHash(height)
|
|
|
|
if err != nil {
|
|
|
|
return false, err
|
|
|
|
}
|
|
|
|
remote, err := chain.GetBlockHash(height)
|
|
|
|
if err != nil {
|
|
|
|
return false, err
|
|
|
|
}
|
|
|
|
if local != remote {
|
|
|
|
return false, nil
|
|
|
|
}
|
|
|
|
return true, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
type blockResult struct {
|
|
|
|
block *Block
|
|
|
|
err error
|
2017-08-28 09:50:57 -06:00
|
|
|
}
|
2017-09-11 07:06:16 -06:00
|
|
|
|
|
|
|
func getBlockChain(
|
|
|
|
hash string,
|
2017-10-05 06:35:07 -06:00
|
|
|
chain Blockchain,
|
2017-09-11 07:06:16 -06:00
|
|
|
out chan blockResult,
|
|
|
|
done chan struct{},
|
|
|
|
) {
|
|
|
|
defer close(out)
|
|
|
|
|
|
|
|
for hash != "" {
|
|
|
|
select {
|
|
|
|
case <-done:
|
|
|
|
return
|
|
|
|
default:
|
|
|
|
}
|
2017-10-05 06:35:07 -06:00
|
|
|
block, err := chain.GetBlock(hash)
|
2017-09-11 07:06:16 -06:00
|
|
|
if err != nil {
|
|
|
|
out <- blockResult{err: err}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
hash = block.Next
|
|
|
|
out <- blockResult{block: block}
|
|
|
|
}
|
|
|
|
}
|