Add dbcache configuration flag, tune db options

blockbook.go

@@ -38,7 +38,8 @@ const storeInternalStatePeriodMs = 59699
 var (
 	blockchain = flag.String("blockchaincfg", "", "path to blockchain RPC service configuration json file")
 
-	dbPath = flag.String("datadir", "./data", "path to database directory")
+	dbPath  = flag.String("datadir", "./data", "path to database directory")
+	dbCache = flag.Int("dbcache", 1<<30, "size of the rocksdb cache")
 
 	blockFrom      = flag.Int("blockheight", -1, "height of the starting block")
 	blockUntil     = flag.Int("blockuntil", -1, "height of the final block")
@@ -162,7 +163,7 @@ func main() {
 		glog.Fatal("rpc: ", err)
 	}
 
-	index, err = db.NewRocksDB(*dbPath, chain.GetChainParser(), metrics)
+	index, err = db.NewRocksDB(*dbPath, *dbCache, chain.GetChainParser(), metrics)
 	if err != nil {
 		glog.Fatal("rocksDB: ", err)
 	}

configs/coins/bitcoin.json

@@ -47,8 +47,8 @@
     "system_user": "blockbook-bitcoin",
     "internal_binding_template": ":{{.Ports.BlockbookInternal}}",
     "public_binding_template": ":{{.Ports.BlockbookPublic}}",
-    "explorer_url": "https://btc-explorer.trezor.io/",
-    "additional_params": "",
+    "explorer_url": "/explorer",
+    "additional_params": "-dbcache=4294967296",
     "block_chain": {
       "parse": true,
       "mempool_workers": 8,

configs/coins/bitcoin_testnet.json

@@ -47,7 +47,7 @@
     "system_user": "blockbook-bitcoin",
     "internal_binding_template": ":{{.Ports.BlockbookInternal}}",
     "public_binding_template": ":{{.Ports.BlockbookPublic}}",
-    "explorer_url": "https://btc-testnet-explorer.trezor.io/",
+    "explorer_url": "/explorer",
     "additional_params": "",
     "block_chain": {
       "parse": true,

db/bulkconnect.go

@@ -31,10 +31,10 @@ type BulkConnect struct {
 }
 
 const (
-	maxBulkAddresses      = 400000
-	maxBulkTxAddresses    = 1500000
+	maxBulkAddresses      = 200000
+	maxBulkTxAddresses    = 500000
 	partialStoreAddresses = maxBulkTxAddresses / 10
-	maxBulkBalances       = 2500000
+	maxBulkBalances       = 800000
 	partialStoreBalances  = maxBulkBalances / 10
 )
 

db/dboptions.go

@@ -1,47 +1,51 @@
 package db
 
-// #include "rocksdb/c.h"
-import "C"
 import (
-	"reflect"
-	"unsafe"
-
 	"github.com/tecbot/gorocksdb"
 )
 
-func createAndSetDBOptions(bloomBits int, twoLevelIndex bool, c *gorocksdb.Cache) *gorocksdb.Options {
-	// get access to all db options, gorocksdb does not expose all
+/*
+	possible additional tuning, using options not accessible by gorocksdb
+
+// #include "rocksdb/c.h"
+import "C"
 
-	// opts := gorocksdb.NewDefaultOptions()
 	cNativeOpts := C.rocksdb_options_create()
 	opts := &gorocksdb.Options{}
 	cField := reflect.Indirect(reflect.ValueOf(opts)).FieldByName("c")
 	cPtr := (**C.rocksdb_options_t)(unsafe.Pointer(cField.UnsafeAddr()))
 	*cPtr = cNativeOpts
 
-	// blockOpts := gorocksdb.NewDefaultBlockBasedTableOptions()
 	cNativeBlockOpts := C.rocksdb_block_based_options_create()
 	blockOpts := &gorocksdb.BlockBasedTableOptions{}
 	cBlockField := reflect.Indirect(reflect.ValueOf(blockOpts)).FieldByName("c")
 	cBlockPtr := (**C.rocksdb_block_based_table_options_t)(unsafe.Pointer(cBlockField.UnsafeAddr()))
 	*cBlockPtr = cNativeBlockOpts
 
+	// https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters
+	blockOpts.SetIndexType(gorocksdb.KTwoLevelIndexSearchIndexType)
+	C.rocksdb_block_based_options_set_partition_filters(cNativeBlockOpts, boolToChar(true))
+	C.rocksdb_block_based_options_set_metadata_block_size(cNativeBlockOpts, C.uint64_t(4096))
+	C.rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority(cNativeBlockOpts, boolToChar(true))
+	blockOpts.SetPinL0FilterAndIndexBlocksInCache(true)
+
+// boolToChar converts a bool value to C.uchar.
+func boolToChar(b bool) C.uchar {
+	if b {
+		return 1
+	}
+	return 0
+}
+*/
+
+func createAndSetDBOptions(bloomBits int, c *gorocksdb.Cache) *gorocksdb.Options {
+	blockOpts := gorocksdb.NewDefaultBlockBasedTableOptions()
 	blockOpts.SetBlockSize(32 << 10) // 32kB
 	blockOpts.SetBlockCache(c)
 	if bloomBits > 0 {
 		blockOpts.SetFilterPolicy(gorocksdb.NewBloomFilter(bloomBits))
 	}
-
-	// https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters
-	if twoLevelIndex {
-		// blockOpts.SetCacheIndexAndFilterBlocks(true)
-		blockOpts.SetIndexType(gorocksdb.KTwoLevelIndexSearchIndexType)
-		C.rocksdb_block_based_options_set_partition_filters(cNativeBlockOpts, boolToChar(true))
-		C.rocksdb_block_based_options_set_metadata_block_size(cNativeBlockOpts, C.uint64_t(4096))
-		C.rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority(cNativeBlockOpts, boolToChar(true))
-		blockOpts.SetPinL0FilterAndIndexBlocksInCache(true)
-	}
-
+	opts := gorocksdb.NewDefaultOptions()
 	opts.SetBlockBasedTableFactory(blockOpts)
 	opts.SetCreateIfMissing(true)
 	opts.SetCreateIfMissingColumnFamilies(true)
@@ -52,14 +56,5 @@ func createAndSetDBOptions(bloomBits int, twoLevelIndex bool, c *gorocksdb.Cache
 	opts.SetMaxBytesForLevelBase(1 << 27) // 128MB
 	opts.SetMaxOpenFiles(25000)
 	opts.SetCompression(gorocksdb.LZ4HCCompression)
-
 	return opts
 }
-
-// boolToChar converts a bool value to C.uchar.
-func boolToChar(b bool) C.uchar {
-	if b {
-		return 1
-	}
-	return 0
-}

db/rocksdb.go

@@ -60,10 +60,10 @@ var cfNames = []string{"default", "height", "addresses", "txAddresses", "address
 
 func openDB(path string, c *gorocksdb.Cache) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) {
 	// opts with bloom filter
-	opts := createAndSetDBOptions(10, true, c)
+	opts := createAndSetDBOptions(10, c)
 	// opts for addresses without bloom filter
 	// from documentation: if most of your queries are executed using iterators, you shouldn't set bloom filter
-	optsAddresses := createAndSetDBOptions(0, true, c)
+	optsAddresses := createAndSetDBOptions(0, c)
 	// default, height, addresses, txAddresses, addressBalance, blockTxids, transactions
 	fcOptions := []*gorocksdb.Options{opts, opts, optsAddresses, opts, opts, opts, opts}
 	db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions)
@@ -75,9 +75,9 @@ func openDB(path string, c *gorocksdb.Cache) (*gorocksdb.DB, []*gorocksdb.Column
 
 // NewRocksDB opens an internal handle to RocksDB environment.  Close
 // needs to be called to release it.
-func NewRocksDB(path string, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) {
+func NewRocksDB(path string, cacheSize int, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) {
 	glog.Infof("rocksdb: open %s, version %v", path, dbVersion)
-	c := gorocksdb.NewLRUCache(1 << 30) // 1GB
+	c := gorocksdb.NewLRUCache(cacheSize)
 	db, cfh, err := openDB(path, c)
 	wo := gorocksdb.NewDefaultWriteOptions()
 	ro := gorocksdb.NewDefaultReadOptions()

db/rocksdb_test.go

@@ -37,7 +37,7 @@ func setupRocksDB(t *testing.T, p bchain.BlockChainParser) *RocksDB {
 	if err != nil {
 		t.Fatal(err)
 	}
-	d, err := NewRocksDB(tmp, p, nil)
+	d, err := NewRocksDB(tmp, 100000, p, nil)
 	if err != nil {
 		t.Fatal(err)
 	}