trie, core/state: improve memory usage and performance (#3135)

* trie: store nodes as pointers

This avoids memory copies when unwrapping node interface values.

name      old time/op  new time/op  delta
Get        388ns ± 8%   215ns ± 2%  -44.56%  (p=0.000 n=15+15)
GetDB      363ns ± 3%   202ns ± 2%  -44.21%  (p=0.000 n=15+15)
UpdateBE  1.57µs ± 2%  1.29µs ± 3%  -17.80%  (p=0.000 n=13+15)
UpdateLE  1.92µs ± 2%  1.61µs ± 2%  -16.25%  (p=0.000 n=14+14)
HashBE    2.16µs ± 6%  2.18µs ± 6%     ~     (p=0.436 n=15+15)
HashLE    7.43µs ± 3%  7.21µs ± 3%   -2.96%  (p=0.000 n=15+13)

* trie: close temporary databases in GetDB benchmark

* trie: don't keep []byte from DB load around

Nodes decoded from a DB load kept hashes and values as sub-slices of
the DB value. This can be a problem because loading from leveldb often
returns []byte with a cap that's larger than necessary, increasing
memory usage.

* trie: unload old cached nodes

* trie, core/state: use cache unloading for account trie

* trie: use explicit private flags (fixes Go 1.5 reflection issue).

* trie: fixup cachegen overflow at request of nick

* core/state: rename journal size constant

Author: fjl

core/blockchain.go

@@ -269,7 +269,7 @@ func (self *BlockChain) FastSyncCommitHead(hash common.Hash) error {
 	if block == nil {
 		return fmt.Errorf("non existent block [%x…]", hash[:4])
 	}
-	if _, err := trie.NewSecure(block.Root(), self.chainDb); err != nil {
+	if _, err := trie.NewSecure(block.Root(), self.chainDb, 0); err != nil {
 		return err
 	}
 	// If all checks out, manually set the head block

core/state/state_object.go

@@ -137,9 +137,9 @@ func (self *StateObject) markSuicided() {
 func (c *StateObject) getTrie(db trie.Database) *trie.SecureTrie {
 	if c.trie == nil {
 		var err error
-		c.trie, err = trie.NewSecure(c.data.Root, db)
+		c.trie, err = trie.NewSecure(c.data.Root, db, 0)
 		if err != nil {
-			c.trie, _ = trie.NewSecure(common.Hash{}, db)
+			c.trie, _ = trie.NewSecure(common.Hash{}, db, 0)
 			c.setError(fmt.Errorf("can't create storage trie: %v", err))
 		}
 	}

core/state/statedb.go

@@ -41,7 +41,10 @@ var StartingNonce uint64
 const (
 	// Number of past tries to keep. The arbitrarily chosen value here
 	// is max uncle depth + 1.
-	maxTrieCacheLength = 8
+	maxPastTries = 8
+
+	// Trie cache generation limit.
+	maxTrieCacheGen = 100
 
 	// Number of codehash->size associations to keep.
 	codeSizeCacheSize = 100000
@@ -86,7 +89,7 @@ type StateDB struct {
 
 // Create a new state from a given trie
 func New(root common.Hash, db ethdb.Database) (*StateDB, error) {
-	tr, err := trie.NewSecure(root, db)
+	tr, err := trie.NewSecure(root, db, maxTrieCacheGen)
 	if err != nil {
 		return nil, err
 	}
@@ -155,14 +158,14 @@ func (self *StateDB) openTrie(root common.Hash) (*trie.SecureTrie, error) {
 			return &tr, nil
 		}
 	}
-	return trie.NewSecure(root, self.db)
+	return trie.NewSecure(root, self.db, maxTrieCacheGen)
 }
 
 func (self *StateDB) pushTrie(t *trie.SecureTrie) {
 	self.lock.Lock()
 	defer self.lock.Unlock()
 
-	if len(self.pastTries) >= maxTrieCacheLength {
+	if len(self.pastTries) >= maxPastTries {
 		copy(self.pastTries, self.pastTries[1:])
 		self.pastTries[len(self.pastTries)-1] = t
 	} else {

eth/downloader/downloader_test.go

@@ -286,7 +286,7 @@ func (dl *downloadTester) headFastBlock() *types.Block {
 func (dl *downloadTester) commitHeadBlock(hash common.Hash) error {
 	// For now only check that the state trie is correct
 	if block := dl.getBlock(hash); block != nil {
-		_, err := trie.NewSecure(block.Root(), dl.stateDb)
+		_, err := trie.NewSecure(block.Root(), dl.stateDb, 0)
 		return err
 	}
 	return fmt.Errorf("non existent block: %x", hash[:4])

light/trie.go

@@ -79,7 +79,7 @@ func (t *LightTrie) do(ctx context.Context, fallbackKey []byte, fn func() error)
 func (t *LightTrie) Get(ctx context.Context, key []byte) (res []byte, err error) {
 	err = t.do(ctx, key, func() (err error) {
 		if t.trie == nil {
-			t.trie, err = trie.NewSecure(t.originalRoot, t.db)
+			t.trie, err = trie.NewSecure(t.originalRoot, t.db, 0)
 		}
 		if err == nil {
 			res, err = t.trie.TryGet(key)
@@ -98,7 +98,7 @@ func (t *LightTrie) Get(ctx context.Context, key []byte) (res []byte, err error)
 func (t *LightTrie) Update(ctx context.Context, key, value []byte) (err error) {
 	err = t.do(ctx, key, func() (err error) {
 		if t.trie == nil {
-			t.trie, err = trie.NewSecure(t.originalRoot, t.db)
+			t.trie, err = trie.NewSecure(t.originalRoot, t.db, 0)
 		}
 		if err == nil {
 			err = t.trie.TryUpdate(key, value)
@@ -112,7 +112,7 @@ func (t *LightTrie) Update(ctx context.Context, key, value []byte) (err error) {
 func (t *LightTrie) Delete(ctx context.Context, key []byte) (err error) {
 	err = t.do(ctx, key, func() (err error) {
 		if t.trie == nil {
-			t.trie, err = trie.NewSecure(t.originalRoot, t.db)
+			t.trie, err = trie.NewSecure(t.originalRoot, t.db, 0)
 		}
 		if err == nil {
 			err = t.trie.TryDelete(key)

trie/hasher.go

@@ -27,8 +27,9 @@ import (
 )
 
 type hasher struct {
-	tmp *bytes.Buffer
-	sha hash.Hash
+	tmp                  *bytes.Buffer
+	sha                  hash.Hash
+	cachegen, cachelimit uint16
 }
 
 // hashers live in a global pool.
@@ -38,8 +39,10 @@ var hasherPool = sync.Pool{
 	},
 }
 
-func newHasher() *hasher {
-	return hasherPool.Get().(*hasher)
+func newHasher(cachegen, cachelimit uint16) *hasher {
+	h := hasherPool.Get().(*hasher)
+	h.cachegen, h.cachelimit = cachegen, cachelimit
+	return h
 }
 
 func returnHasherToPool(h *hasher) {
@@ -50,8 +53,18 @@ func returnHasherToPool(h *hasher) {
 // original node initialzied with the computed hash to replace the original one.
 func (h *hasher) hash(n node, db DatabaseWriter, force bool) (node, node, error) {
 	// If we're not storing the node, just hashing, use avaialble cached data
-	if hash, dirty := n.cache(); hash != nil && (db == nil || !dirty) {
-		return hash, n, nil
+	if hash, dirty := n.cache(); hash != nil {
+		if db == nil {
+			return hash, n, nil
+		}
+		if n.canUnload(h.cachegen, h.cachelimit) {
+			// Evict the node from cache. All of its subnodes will have a lower or equal
+			// cache generation number.
+			return hash, hash, nil
+		}
+		if !dirty {
+			return hash, n, nil
+		}
 	}
 	// Trie not processed yet or needs storage, walk the children
 	collapsed, cached, err := h.hashChildren(n, db)
@@ -62,19 +75,21 @@ func (h *hasher) hash(n node, db DatabaseWriter, force bool) (node, node, error)
 	if err != nil {
 		return hashNode{}, n, err
 	}
-	// Cache the hash and RLP blob of the ndoe for later reuse
+	// Cache the hash of the ndoe for later reuse.
 	if hash, ok := hashed.(hashNode); ok && !force {
 		switch cached := cached.(type) {
-		case shortNode:
-			cached.hash = hash
+		case *shortNode:
+			cached = cached.copy()
+			cached.flags.hash = hash
 			if db != nil {
-				cached.dirty = false
+				cached.flags.dirty = false
 			}
 			return hashed, cached, nil
-		case fullNode:
-			cached.hash = hash
+		case *fullNode:
+			cached = cached.copy()
+			cached.flags.hash = hash
 			if db != nil {
-				cached.dirty = false
+				cached.flags.dirty = false
 			}
 			return hashed, cached, nil
 		}
@@ -89,40 +104,42 @@ func (h *hasher) hashChildren(original node, db DatabaseWriter) (node, node, err
 	var err error
 
 	switch n := original.(type) {
-	case shortNode:
+	case *shortNode:
 		// Hash the short node's child, caching the newly hashed subtree
-		cached := n
-		cached.Key = common.CopyBytes(cached.Key)
+		collapsed, cached := n.copy(), n.copy()
+		collapsed.Key = compactEncode(n.Key)
+		cached.Key = common.CopyBytes(n.Key)
 
-		n.Key = compactEncode(n.Key)
 		if _, ok := n.Val.(valueNode); !ok {
-			if n.Val, cached.Val, err = h.hash(n.Val, db, false); err != nil {
-				return n, original, err
+			collapsed.Val, cached.Val, err = h.hash(n.Val, db, false)
+			if err != nil {
+				return original, original, err
 			}
 		}
-		if n.Val == nil {
-			n.Val = valueNode(nil) // Ensure that nil children are encoded as empty strings.
+		if collapsed.Val == nil {
+			collapsed.Val = valueNode(nil) // Ensure that nil children are encoded as empty strings.
 		}
-		return n, cached, nil
+		return collapsed, cached, nil
 
-	case fullNode:
+	case *fullNode:
 		// Hash the full node's children, caching the newly hashed subtrees
-		cached := fullNode{dirty: n.dirty}
+		collapsed, cached := n.copy(), n.copy()
 
 		for i := 0; i < 16; i++ {
 			if n.Children[i] != nil {
-				if n.Children[i], cached.Children[i], err = h.hash(n.Children[i], db, false); err != nil {
-					return n, original, err
+				collapsed.Children[i], cached.Children[i], err = h.hash(n.Children[i], db, false)
+				if err != nil {
+					return original, original, err
 				}
 			} else {
-				n.Children[i] = valueNode(nil) // Ensure that nil children are encoded as empty strings.
+				collapsed.Children[i] = valueNode(nil) // Ensure that nil children are encoded as empty strings.
 			}
 		}
 		cached.Children[16] = n.Children[16]
-		if n.Children[16] == nil {
-			n.Children[16] = valueNode(nil)
+		if collapsed.Children[16] == nil {
+			collapsed.Children[16] = valueNode(nil)
 		}
-		return n, cached, nil
+		return collapsed, cached, nil
 
 	default:
 		// Value and hash nodes don't have children so they're left as were
@@ -140,6 +157,7 @@ func (h *hasher) store(n node, db DatabaseWriter, force bool) (node, error) {
 	if err := rlp.Encode(h.tmp, n); err != nil {
 		panic("encode error: " + err.Error())
 	}
+
 	if h.tmp.Len() < 32 && !force {
 		return n, nil // Nodes smaller than 32 bytes are stored inside their parent
 	}

trie/iterator.go

@@ -58,11 +58,11 @@ func (it *Iterator) makeKey() []byte {
 	key := it.keyBuf[:0]
 	for _, se := range it.nodeIt.stack {
 		switch node := se.node.(type) {
-		case fullNode:
+		case *fullNode:
 			if se.child <= 16 {
 				key = append(key, byte(se.child))
 			}
-		case shortNode:
+		case *shortNode:
 			if hasTerm(node.Key) {
 				key = append(key, node.Key[:len(node.Key)-1]...)
 			} else {
@@ -150,30 +150,30 @@ func (it *NodeIterator) step() error {
 		if (ancestor == common.Hash{}) {
 			ancestor = parent.parent
 		}
-		if node, ok := parent.node.(fullNode); ok {
+		if node, ok := parent.node.(*fullNode); ok {
 			// Full node, traverse all children, then the node itself
 			if parent.child >= len(node.Children) {
 				break
 			}
 			for parent.child++; parent.child < len(node.Children); parent.child++ {
 				if current := node.Children[parent.child]; current != nil {
 					it.stack = append(it.stack, &nodeIteratorState{
-						hash:   common.BytesToHash(node.hash),
+						hash:   common.BytesToHash(node.flags.hash),
 						node:   current,
 						parent: ancestor,
 						child:  -1,
 					})
 					break
 				}
 			}
-		} else if node, ok := parent.node.(shortNode); ok {
+		} else if node, ok := parent.node.(*shortNode); ok {
 			// Short node, traverse the pointer singleton child, then the node itself
 			if parent.child >= 0 {
 				break
 			}
 			parent.child++
 			it.stack = append(it.stack, &nodeIteratorState{
-				hash:   common.BytesToHash(node.hash),
+				hash:   common.BytesToHash(node.flags.hash),
 				node:   node.Val,
 				parent: ancestor,
 				child:  -1,