configurable concurrency, dry run
parent
4c33e066c8
commit
c2761de9ed
23
blockbook.go
23
blockbook.go
|
@ -44,6 +44,10 @@ var (
|
||||||
resync = flag.Bool("resync", false, "resync until tip")
|
resync = flag.Bool("resync", false, "resync until tip")
|
||||||
repair = flag.Bool("repair", false, "repair the database")
|
repair = flag.Bool("repair", false, "repair the database")
|
||||||
prof = flag.Bool("prof", false, "profile program execution")
|
prof = flag.Bool("prof", false, "profile program execution")
|
||||||
|
|
||||||
|
syncChunk = flag.Int("chunk", 100, "block chunk size for processing")
|
||||||
|
syncWorkers = flag.Int("workers", 8, "number of workers to process blocks")
|
||||||
|
dryRun = flag.Bool("dryrun", false, "do not index blocks, only download")
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
|
@ -91,7 +95,14 @@ func main() {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if err = connectBlocksParallel(rpc, db, height, until); err != nil {
|
if err = connectBlocksParallel(
|
||||||
|
rpc,
|
||||||
|
db,
|
||||||
|
height,
|
||||||
|
until,
|
||||||
|
*syncChunk,
|
||||||
|
*syncWorkers,
|
||||||
|
); err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -193,10 +204,9 @@ func connectBlocksParallel(
|
||||||
index Index,
|
index Index,
|
||||||
lower uint32,
|
lower uint32,
|
||||||
higher uint32,
|
higher uint32,
|
||||||
|
chunkSize int,
|
||||||
|
numWorkers int,
|
||||||
) error {
|
) error {
|
||||||
const chunkSize = 100
|
|
||||||
const numWorkers = 8
|
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
work := func(i int) {
|
work := func(i int) {
|
||||||
|
@ -206,7 +216,7 @@ func connectBlocksParallel(
|
||||||
stride := uint32(chunkSize * numWorkers)
|
stride := uint32(chunkSize * numWorkers)
|
||||||
|
|
||||||
for low := offset; low <= higher; low += stride {
|
for low := offset; low <= higher; low += stride {
|
||||||
high := low + chunkSize - 1
|
high := low + uint32(chunkSize-1)
|
||||||
if high > higher {
|
if high > higher {
|
||||||
high = higher
|
high = higher
|
||||||
}
|
}
|
||||||
|
@ -249,6 +259,9 @@ func connectBlockChunk(
|
||||||
}
|
}
|
||||||
hash = block.Next
|
hash = block.Next
|
||||||
height = block.Height + 1
|
height = block.Height + 1
|
||||||
|
if *dryRun {
|
||||||
|
continue
|
||||||
|
}
|
||||||
err = index.ConnectBlock(block)
|
err = index.ConnectBlock(block)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|
Loading…
Reference in New Issue