From 184f2e9ca6f73efd75e6173e374c37ef19ce05d3 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Tue, 18 Sep 2018 11:49:39 +0200 Subject: [PATCH] Add maxOpenFiles rocksdb flag --- blockbook.go | 7 ++++--- db/dboptions.go | 4 ++-- db/rocksdb.go | 35 ++++++++++++++++++----------------- db/rocksdb_test.go | 2 +- 4 files changed, 25 insertions(+), 23 deletions(-) diff --git a/blockbook.go b/blockbook.go index 6af45abf..9320cc53 100644 --- a/blockbook.go +++ b/blockbook.go @@ -36,8 +36,9 @@ const storeInternalStatePeriodMs = 59699 var ( blockchain = flag.String("blockchaincfg", "", "path to blockchain RPC service configuration json file") - dbPath = flag.String("datadir", "./data", "path to database directory") - dbCache = flag.Int("dbcache", 1<<29, "size of the rocksdb cache") + dbPath = flag.String("datadir", "./data", "path to database directory") + dbCache = flag.Int("dbcache", 1<<29, "size of the rocksdb cache") + dbMaxOpenFiles = flag.Int("dbmaxopenfiles", 1<<14, "max open files by rocksdb") blockFrom = flag.Int("blockheight", -1, "height of the starting block") blockUntil = flag.Int("blockuntil", -1, "height of the final block") @@ -165,7 +166,7 @@ func main() { glog.Fatal("rpc: ", err) } - index, err = db.NewRocksDB(*dbPath, *dbCache, chain.GetChainParser(), metrics) + index, err = db.NewRocksDB(*dbPath, *dbCache, *dbMaxOpenFiles, chain.GetChainParser(), metrics) if err != nil { glog.Fatal("rocksDB: ", err) } diff --git a/db/dboptions.go b/db/dboptions.go index 2e0da942..f982f8a3 100644 --- a/db/dboptions.go +++ b/db/dboptions.go @@ -38,7 +38,7 @@ func boolToChar(b bool) C.uchar { } */ -func createAndSetDBOptions(bloomBits int, c *gorocksdb.Cache) *gorocksdb.Options { +func createAndSetDBOptions(bloomBits int, c *gorocksdb.Cache, maxOpenFiles int) *gorocksdb.Options { blockOpts := gorocksdb.NewDefaultBlockBasedTableOptions() blockOpts.SetBlockSize(32 << 10) // 32kB blockOpts.SetBlockCache(c) @@ -54,7 +54,7 @@ func createAndSetDBOptions(bloomBits int, c *gorocksdb.Cache) *gorocksdb.Options opts.SetBytesPerSync(8 << 20) // 8MB opts.SetWriteBufferSize(1 << 27) // 128MB opts.SetMaxBytesForLevelBase(1 << 27) // 128MB - opts.SetMaxOpenFiles(25000) + opts.SetMaxOpenFiles(maxOpenFiles) opts.SetCompression(gorocksdb.LZ4HCCompression) return opts } diff --git a/db/rocksdb.go b/db/rocksdb.go index 8b715830..81f0a6cb 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -35,15 +35,16 @@ func RepairRocksDB(name string) error { // RocksDB handle type RocksDB struct { - path string - db *gorocksdb.DB - wo *gorocksdb.WriteOptions - ro *gorocksdb.ReadOptions - cfh []*gorocksdb.ColumnFamilyHandle - chainParser bchain.BlockChainParser - is *common.InternalState - metrics *common.Metrics - cache *gorocksdb.Cache + path string + db *gorocksdb.DB + wo *gorocksdb.WriteOptions + ro *gorocksdb.ReadOptions + cfh []*gorocksdb.ColumnFamilyHandle + chainParser bchain.BlockChainParser + is *common.InternalState + metrics *common.Metrics + cache *gorocksdb.Cache + maxOpenFiles int } const ( @@ -58,12 +59,12 @@ const ( var cfNames = []string{"default", "height", "addresses", "txAddresses", "addressBalance", "blockTxs", "transactions"} -func openDB(path string, c *gorocksdb.Cache) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { +func openDB(path string, c *gorocksdb.Cache, openFiles int) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { // opts with bloom filter - opts := createAndSetDBOptions(10, c) + opts := createAndSetDBOptions(10, c, openFiles) // opts for addresses without bloom filter // from documentation: if most of your queries are executed using iterators, you shouldn't set bloom filter - optsAddresses := createAndSetDBOptions(0, c) + optsAddresses := createAndSetDBOptions(0, c, openFiles) // default, height, addresses, txAddresses, addressBalance, blockTxids, transactions fcOptions := []*gorocksdb.Options{opts, opts, optsAddresses, opts, opts, opts, opts} db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions) @@ -75,13 +76,13 @@ func openDB(path string, c *gorocksdb.Cache) (*gorocksdb.DB, []*gorocksdb.Column // NewRocksDB opens an internal handle to RocksDB environment. Close // needs to be called to release it. -func NewRocksDB(path string, cacheSize int, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) { - glog.Infof("rocksdb: open %s, version %v", path, dbVersion) +func NewRocksDB(path string, cacheSize, maxOpenFiles int, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) { + glog.Infof("rocksdb: opening %s, required data version %v, cache size %v, max open files %v", path, dbVersion, cacheSize, maxOpenFiles) c := gorocksdb.NewLRUCache(cacheSize) - db, cfh, err := openDB(path, c) + db, cfh, err := openDB(path, c, maxOpenFiles) wo := gorocksdb.NewDefaultWriteOptions() ro := gorocksdb.NewDefaultReadOptions() - return &RocksDB{path, db, wo, ro, cfh, parser, nil, metrics, c}, nil + return &RocksDB{path, db, wo, ro, cfh, parser, nil, metrics, c, maxOpenFiles}, nil } func (d *RocksDB) closeDB() error { @@ -119,7 +120,7 @@ func (d *RocksDB) Reopen() error { return err } d.db = nil - db, cfh, err := openDB(d.path, d.cache) + db, cfh, err := openDB(d.path, d.cache, d.maxOpenFiles) if err != nil { return err } diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 34ede351..8e114597 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -36,7 +36,7 @@ func setupRocksDB(t *testing.T, p bchain.BlockChainParser) *RocksDB { if err != nil { t.Fatal(err) } - d, err := NewRocksDB(tmp, 100000, p, nil) + d, err := NewRocksDB(tmp, 100000, -1, p, nil) if err != nil { t.Fatal(err) }