Merge pull request #2228 from karalabe/allocate-cache-to-chaindata

cmd, eth, ethdb, node: prioritise chaindata for resources, bump cache

cmd/utils/fdlimit_test.go

@@ -0,0 +1,35 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package utils
+
+import "testing"
+
+// TestFileDescriptorLimits simply tests whether the file descriptor allowance
+// per this process can be retrieved.
+func TestFileDescriptorLimits(t *testing.T) {
+	target := 4096
+
+	if limit, err := getFdLimit(); err != nil || limit <= 0 {
+		t.Fatalf("failed to retrieve file descriptor limit (%d): %v", limit, err)
+	}
+	if err := raiseFdLimit(uint64(target)); err != nil {
+		t.Fatalf("failed to raise file allowance")
+	}
+	if limit, err := getFdLimit(); err != nil || limit < target {
+		t.Fatalf("failed to retrieve raised descriptor limit (have %v, want %v): %v", limit, target, err)
+	}
+}

cmd/utils/fdlimit_unix.go

@@ -0,0 +1,50 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// +build linux darwin
+
+package utils
+
+import "syscall"
+
+// raiseFdLimit tries to maximize the file descriptor allowance of this process
+// to the maximum hard-limit allowed by the OS.
+func raiseFdLimit(max uint64) error {
+	// Get the current limit
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return err
+	}
+	// Try to update the limit to the max allowance
+	limit.Cur = limit.Max
+	if limit.Cur > max {
+		limit.Cur = max
+	}
+	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return err
+	}
+	return nil
+}
+
+// getFdLimit retrieves the number of file descriptors allowed to be opened by this
+// process.
+func getFdLimit() (int, error) {
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	return int(limit.Cur), nil
+}

cmd/utils/fdlimit_windows.go

@@ -0,0 +1,41 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package utils
+
+import "errors"
+
+// raiseFdLimit tries to maximize the file descriptor allowance of this process
+// to the maximum hard-limit allowed by the OS.
+func raiseFdLimit(max uint64) error {
+	// This method is NOP by design:
+	//  * Linux/Darwin counterparts need to manually increase per process limits
+	//  * On Windows Go uses the CreateFile API, which is limited to 16K files, non
+	//    changeable from within a running process
+	// This way we can always "request" raising the limits, which will either have
+	// or not have effect based on the platform we're running on.
+	if max > 16384 {
+		return errors.New("file descriptor limit (16384) reached")
+	}
+	return nil
+}
+
+// getFdLimit retrieves the number of file descriptors allowed to be opened by this
+// process.
+func getFdLimit() (int, error) {
+	// Please see raiseFdLimit for the reason why we use hard coded 16K as the limit
+	return 16384, nil
+}

cmd/utils/flags.go

@@ -143,7 +143,7 @@ var (
 	CacheFlag = cli.IntFlag{
 		Name:  "cache",
 		Usage: "Megabytes of memory allocated to internal caching (min 16MB / database forced)",
-		Value: 0,
+		Value: 128,
 	}
 	BlockchainVersionFlag = cli.IntFlag{
 		Name:  "blockchainversion",
@@ -527,6 +527,22 @@ func MakeGenesisBlock(ctx *cli.Context) string {
 	return string(data)
 }
 
+// MakeDatabaseHandles raises out the number of allowed file handles per process
+// for Geth and returns half of the allowance to assign to the database.
+func MakeDatabaseHandles() int {
+	if err := raiseFdLimit(2048); err != nil {
+		Fatalf("Failed to raise file descriptor allowance: %v", err)
+	}
+	limit, err := getFdLimit()
+	if err != nil {
+		Fatalf("Failed to retrieve file descriptor allowance: %v", err)
+	}
+	if limit > 2048 { // cap database file descriptors even if more is available
+		limit = 2048
+	}
+	return limit / 2 // Leave half for networking and other stuff
+}
+
 // MakeAccountManager creates an account manager from set command line flags.
 func MakeAccountManager(ctx *cli.Context) *accounts.Manager {
 	// Create the keystore crypto primitive, light if requested
@@ -649,6 +665,7 @@ func MakeSystemNode(name, version string, extra []byte, ctx *cli.Context) *node.
 		FastSync:                ctx.GlobalBool(FastSyncFlag.Name),
 		BlockChainVersion:       ctx.GlobalInt(BlockchainVersionFlag.Name),
 		DatabaseCache:           ctx.GlobalInt(CacheFlag.Name),
+		DatabaseHandles:         MakeDatabaseHandles(),
 		NetworkId:               ctx.GlobalInt(NetworkIdFlag.Name),
 		AccountManager:          accman,
 		Etherbase:               MakeEtherbase(accman, ctx),
@@ -763,9 +780,10 @@ func SetupVM(ctx *cli.Context) {
 func MakeChain(ctx *cli.Context) (chain *core.BlockChain, chainDb ethdb.Database) {
 	datadir := MustMakeDataDir(ctx)
 	cache := ctx.GlobalInt(CacheFlag.Name)
+	handles := MakeDatabaseHandles()
 
 	var err error
-	if chainDb, err = ethdb.NewLDBDatabase(filepath.Join(datadir, "chaindata"), cache); err != nil {
+	if chainDb, err = ethdb.NewLDBDatabase(filepath.Join(datadir, "chaindata"), cache, handles); err != nil {
 		Fatalf("Could not open database: %v", err)
 	}
 	if ctx.GlobalBool(OlympicFlag.Name) {

core/bench_test.go

@@ -153,7 +153,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) {
 			b.Fatalf("cannot create temporary directory: %v", err)
 		}
 		defer os.RemoveAll(dir)
-		db, err = ethdb.NewLDBDatabase(dir, 0)
+		db, err = ethdb.NewLDBDatabase(dir, 128, 128)
 		if err != nil {
 			b.Fatalf("cannot create temporary database: %v", err)
 		}

core/database_util_test.go

@@ -551,7 +551,7 @@ func TestMipmapChain(t *testing.T) {
 	defer os.RemoveAll(dir)
 
 	var (
-		db, _   = ethdb.NewLDBDatabase(dir, 16)
+		db, _   = ethdb.NewLDBDatabase(dir, 0, 0)
 		key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
 		addr    = crypto.PubkeyToAddress(key1.PublicKey)
 		addr2   = common.BytesToAddress([]byte("jeff"))

eth/backend.go

@@ -69,6 +69,7 @@ type Config struct {
 	BlockChainVersion  int
 	SkipBcVersionCheck bool // e.g. blockchain export
 	DatabaseCache      int
+	DatabaseHandles    int
 
 	NatSpec   bool
 	DocRoot   string
@@ -135,12 +136,8 @@ type Ethereum struct {
 }
 
 func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
-	// Let the database take 3/4 of the max open files (TODO figure out a way to get the actual limit of the open files)
-	const dbCount = 3
-	ethdb.OpenFileLimit = 128 / (dbCount + 1)
-
 	// Open the chain database and perform any upgrades needed
-	chainDb, err := ctx.OpenDatabase("chaindata", config.DatabaseCache)
+	chainDb, err := ctx.OpenDatabase("chaindata", config.DatabaseCache, config.DatabaseHandles)
 	if err != nil {
 		return nil, err
 	}
@@ -154,7 +151,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
 		return nil, err
 	}
 
-	dappDb, err := ctx.OpenDatabase("dapp", config.DatabaseCache)
+	dappDb, err := ctx.OpenDatabase("dapp", config.DatabaseCache, config.DatabaseHandles)
 	if err != nil {
 		return nil, err
 	}

eth/filters/filter_test.go

@@ -31,7 +31,7 @@ func BenchmarkMipmaps(b *testing.B) {
 	defer os.RemoveAll(dir)
 
 	var (
-		db, _   = ethdb.NewLDBDatabase(dir, 16)
+		db, _   = ethdb.NewLDBDatabase(dir, 0, 0)
 		key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
 		addr1   = crypto.PubkeyToAddress(key1.PublicKey)
 		addr2   = common.BytesToAddress([]byte("jeff"))
@@ -105,7 +105,7 @@ func TestFilters(t *testing.T) {
 	defer os.RemoveAll(dir)
 
 	var (
-		db, _   = ethdb.NewLDBDatabase(dir, 16)
+		db, _   = ethdb.NewLDBDatabase(dir, 0, 0)
 		key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
 		addr    = crypto.PubkeyToAddress(key1.PublicKey)
 

ethdb/database.go

@@ -39,8 +39,15 @@ var OpenFileLimit = 64
 // cacheRatio specifies how the total alloted cache is distributed between the
 // various system databases.
 var cacheRatio = map[string]float64{
-	"dapp":      2.0 / 13.0,
-	"chaindata": 11.0 / 13.0,
+	"dapp":      0.0,
+	"chaindata": 1.0,
+}
+
+// handleRatio specifies how the total alloted file descriptors is distributed
+// between the various system databases.
+var handleRatio = map[string]float64{
+	"dapp":      0.0,
+	"chaindata": 1.0,
 }
 
 type LDBDatabase struct {
@@ -62,17 +69,21 @@ type LDBDatabase struct {
 }
 
 // NewLDBDatabase returns a LevelDB wrapped object.
-func NewLDBDatabase(file string, cache int) (*LDBDatabase, error) {
-	// Calculate the cache allowance for this particular database
+func NewLDBDatabase(file string, cache int, handles int) (*LDBDatabase, error) {
+	// Calculate the cache and file descriptor allowance for this particular database
 	cache = int(float64(cache) * cacheRatio[filepath.Base(file)])
 	if cache < 16 {
 		cache = 16
 	}
-	glog.V(logger.Info).Infof("Alloted %dMB cache to %s", cache, file)
+	handles = int(float64(handles) * handleRatio[filepath.Base(file)])
+	if handles < 16 {
+		handles = 16
+	}
+	glog.V(logger.Info).Infof("Alloted %dMB cache and %d file handles to %s", cache, handles, file)
 
 	// Open the db and recover any potential corruptions
 	db, err := leveldb.OpenFile(file, &opt.Options{
-		OpenFilesCacheCapacity: OpenFileLimit,
+		OpenFilesCacheCapacity: handles,
 		BlockCacheCapacity:     cache / 2 * opt.MiB,
 		WriteBuffer:            cache / 4 * opt.MiB, // Two of these are used internally
 	})

ethdb/database_test.go

@@ -28,7 +28,7 @@ func newDb() *LDBDatabase {
 	if common.FileExist(file) {
 		os.RemoveAll(file)
 	}
-	db, _ := NewLDBDatabase(file, 0)
+	db, _ := NewLDBDatabase(file, 0, 0)
 
 	return db
 }

node/service.go

@@ -38,11 +38,11 @@ type ServiceContext struct {
 // OpenDatabase opens an existing database with the given name (or creates one
 // if no previous can be found) from within the node's data directory. If the
 // node is an ephemeral one, a memory database is returned.
-func (ctx *ServiceContext) OpenDatabase(name string, cache int) (ethdb.Database, error) {
+func (ctx *ServiceContext) OpenDatabase(name string, cache int, handles int) (ethdb.Database, error) {
 	if ctx.datadir == "" {
 		return ethdb.NewMemDatabase()
 	}
-	return ethdb.NewLDBDatabase(filepath.Join(ctx.datadir, name), cache)
+	return ethdb.NewLDBDatabase(filepath.Join(ctx.datadir, name), cache, handles)
 }
 
 // Service retrieves a currently running service registered of a specific type.

node/service_test.go

@@ -39,7 +39,7 @@ func TestContextDatabases(t *testing.T) {
 	}
 	// Request the opening/creation of a database and ensure it persists to disk
 	ctx := &ServiceContext{datadir: dir}
-	db, err := ctx.OpenDatabase("persistent", 0)
+	db, err := ctx.OpenDatabase("persistent", 0, 0)
 	if err != nil {
 		t.Fatalf("failed to open persistent database: %v", err)
 	}
@@ -50,7 +50,7 @@ func TestContextDatabases(t *testing.T) {
 	}
 	// Request th opening/creation of an ephemeral database and ensure it's not persisted
 	ctx = &ServiceContext{datadir: ""}
-	db, err = ctx.OpenDatabase("ephemeral", 0)
+	db, err = ctx.OpenDatabase("ephemeral", 0, 0)
 	if err != nil {
 		t.Fatalf("failed to open ephemeral database: %v", err)
 	}

trie/trie_test.go

@@ -461,7 +461,7 @@ func tempDB() (string, Database) {
 	if err != nil {
 		panic(fmt.Sprintf("can't create temporary directory: %v", err))
 	}
-	db, err := ethdb.NewLDBDatabase(dir, 300*1024)
+	db, err := ethdb.NewLDBDatabase(dir, 256, 0)
 	if err != nil {
 		panic(fmt.Sprintf("can't create temporary database: %v", err))
 	}