openatom_foundation
/
bitxhub
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #61 from meshplus/ledger 

refactor(ledger): add interface to remove journal
This commit is contained in:
Aiden X 2020-04-29 16:40:02 +08:00 committed by GitHub
parent 4a167cf715 e72cb6cbab
commit 381798c319
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 192 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
internal/ledger/journal.go
View File

@ -3,12 +3,17 @@ package ledger
import (
"encoding/hex"
"encoding/json"
"fmt"
"strconv"
"github.com/meshplus/bitxhub-kit/types"
"github.com/meshplus/bitxhub/pkg/storage"
)
var (
minHeightStr = "minHeight"
maxHeightStr = "maxHeight"
)
type journal struct {
Address types.Address
PrevAccount *innerAccount
@ -58,28 +63,21 @@ func (journal *journal) revert(batch storage.Batch) {
}
}
func getLatestJournal(ldb storage.Storage) (uint64, *BlockJournal, error) {
func getJournalRange(ldb storage.Storage) (uint64, uint64) {
minHeight := uint64(0)
maxHeight := uint64(0)
journal := &BlockJournal{}
begin, end := bytesPrefix([]byte(journalKey))
it := ldb.Iterator(begin, end)
for it.Next() {
height := uint64(0)
_, err := fmt.Sscanf(string(it.Key()), journalKey+"%d", &height)
if err != nil {
return 0, nil, err
}
if height > maxHeight {
maxHeight = height
if err := json.Unmarshal(it.Value(), journal); err != nil {
panic(err)
}
}
data := ldb.Get(compositeKey(journalKey, minHeightStr))
if data != nil {
minHeight = unmarshalHeight(data)
}
return maxHeight, journal, nil
data = ldb.Get(compositeKey(journalKey, maxHeightStr))
if data != nil {
maxHeight = unmarshalHeight(data)
}
return minHeight, maxHeight
}
func getBlockJournal(height uint64, ldb storage.Storage) *BlockJournal {
@ -95,3 +93,16 @@ func getBlockJournal(height uint64, ldb storage.Storage) *BlockJournal {
return journal
}
func marshalHeight(height uint64) []byte {
return []byte(strconv.FormatUint(height, 10))
}
func unmarshalHeight(data []byte) uint64 {
height, err := strconv.ParseUint(string(data), 10, 64)
if err != nil {
panic(err)
}
return height
}

72
internal/ledger/ledger.go
View File

@ -15,18 +15,21 @@ import (
var _ Ledger = (*ChainLedger)(nil)
var (
ErrorRollbackToHigherNumber = fmt.Errorf("rollback to higher blockchain height")
ErrorRollbackWithoutJournal = fmt.Errorf("rollback to blockchain height without journal")
ErrorRollbackToHigherNumber = fmt.Errorf("rollback to higher blockchain height")
ErrorRollbackWithoutJournal = fmt.Errorf("rollback to blockchain height without journal")
ErrorRollbackTooMuch = fmt.Errorf("rollback too much block")
ErrorRemoveJournalOutOfRange = fmt.Errorf("remove journal out of range")
)
type ChainLedger struct {
logger logrus.FieldLogger
blockchainStore storage.Storage
ldb storage.Storage
height uint64
minJnlHeight uint64
maxJnlHeight uint64
events map[string][]*pb.Event
accounts map[string]*Account
prevJournalHash types.Hash
prevJnlHash types.Hash
chainMutex sync.RWMutex
chainMeta *pb.ChainMeta
@ -44,43 +47,52 @@ func New(repoRoot string, blockchainStore storage.Storage, logger logrus.FieldLo
return nil, fmt.Errorf("load chain meta: %w", err)
}
height, blockJournal, err := getLatestJournal(ldb)
if err != nil {
return nil, fmt.Errorf("get journal height: %w", err)
}
minJnlHeight, maxJnlHeight := getJournalRange(ldb)
if height < chainMeta.Height {
if maxJnlHeight < chainMeta.Height {
// TODO(xcc): how to handle this case
panic("state tree height is less than blockchain height")
}
prevJnlHash := types.Hash{}
if maxJnlHeight != 0 {
blockJournal := getBlockJournal(maxJnlHeight, ldb)
prevJnlHash = blockJournal.ChangedHash
}
return &ChainLedger{
logger: logger,
chainMeta: chainMeta,
blockchainStore: blockchainStore,
ldb: ldb,
height: height,
minJnlHeight: minJnlHeight,
maxJnlHeight: maxJnlHeight,
events: make(map[string][]*pb.Event, 10),
accounts: make(map[string]*Account),
prevJournalHash: blockJournal.ChangedHash,
prevJnlHash: prevJnlHash,
}, nil
}
// Rollback rollback ledger to history version
func (l *ChainLedger) Rollback(height uint64) error {
if l.height < height {
if l.maxJnlHeight < height {
return ErrorRollbackToHigherNumber
}
if l.height == height {
if l.minJnlHeight > height {
return ErrorRollbackTooMuch
}
if l.maxJnlHeight == height {
return nil
}
// clean cache account
l.Clear()
for i := l.height; i > height; i-- {
for i := l.maxJnlHeight; i > height; i-- {
batch := l.ldb.NewBatch()
blockJournal := getBlockJournal(i, l.ldb)
if blockJournal == nil {
return ErrorRollbackWithoutJournal
@ -90,19 +102,37 @@ func (l *ChainLedger) Rollback(height uint64) error {
journal.revert(batch)
}
batch.Delete(compositeKey(journalKey, i))
batch.Put(compositeKey(journalKey, maxHeightStr), marshalHeight(i-1))
batch.Commit()
l.ldb.Delete(compositeKey(journalKey, i))
}
height, journal, err := getLatestJournal(l.ldb)
if err != nil {
return fmt.Errorf("get journal during rollback: %w", err)
journal := getBlockJournal(height, l.ldb)
l.maxJnlHeight = height
l.prevJnlHash = journal.ChangedHash
return nil
}
// RemoveJournalsBeforeBlock removes ledger journals whose block number < height
func (l *ChainLedger) RemoveJournalsBeforeBlock(height uint64) error {
if height > l.maxJnlHeight {
return ErrorRemoveJournalOutOfRange
}
l.prevJournalHash = journal.ChangedHash
if height <= l.minJnlHeight {
return nil
}
l.height = height
batch := l.ldb.NewBatch()
for i := l.minJnlHeight; i < height; i++ {
batch.Delete(compositeKey(journalKey, i))
}
batch.Put(compositeKey(journalKey, minHeightStr), marshalHeight(height))
batch.Commit()
l.minJnlHeight = height
return nil
}

107
internal/ledger/ledger_test.go
View File

@ -44,7 +44,7 @@ func TestLedger_Commit(t *testing.T) {
ledger.SetBalance(account, 100)
hash, err = ledger.Commit(4)
assert.Equal(t, uint64(4), ledger.height)
assert.Equal(t, uint64(4), ledger.maxJnlHeight)
assert.Nil(t, err)
assert.Equal(t, uint64(4), ledger.Version())
assert.Equal(t, "0x8ef7f408372406532c7060045d77fb67d322cea7aa49afdc3a741f4f340dc6d5", hash.Hex())
@ -56,11 +56,13 @@ func TestLedger_Commit(t *testing.T) {
hash, err = ledger.Commit(5)
assert.Nil(t, err)
assert.Equal(t, uint64(5), ledger.Version())
assert.Equal(t, uint64(5), ledger.height)
assert.Equal(t, uint64(5), ledger.maxJnlHeight)
height, journal, err := getLatestJournal(ledger.ldb)
minHeight, maxHeight := getJournalRange(ledger.ldb)
journal := getBlockJournal(maxHeight, ledger.ldb)
assert.Nil(t, err)
assert.Equal(t, uint64(5), height)
assert.Equal(t, uint64(1), minHeight)
assert.Equal(t, uint64(5), maxHeight)
assert.Equal(t, hash, journal.ChangedHash)
assert.Equal(t, 1, len(journal.Journals))
entry := journal.Journals[0]
@ -80,8 +82,8 @@ func TestLedger_Commit(t *testing.T) {
// load ChainLedger from db
ldg, err := New(repoRoot, blockStorage, log.NewWithModule("executor"))
assert.Nil(t, err)
assert.Equal(t, uint64(5), ldg.height)
assert.Equal(t, hash, ldg.prevJournalHash)
assert.Equal(t, uint64(5), ldg.maxJnlHeight)
assert.Equal(t, hash, ldg.prevJnlHash)
ok, value := ldg.GetState(account, []byte("a"))
assert.True(t, ok)
@ -114,6 +116,9 @@ func TestChainLedger_Rollback(t *testing.T) {
addr0 := types.Bytes2Address(bytesutil.LeftPadBytes([]byte{100}, 20))
addr1 := types.Bytes2Address(bytesutil.LeftPadBytes([]byte{101}, 20))
hash0 := types.Hash{}
assert.Equal(t, hash0, ledger.prevJnlHash)
code := sha256.Sum256([]byte("code"))
codeHash := sha256.Sum256(code[:])
@ -140,11 +145,13 @@ func TestChainLedger_Rollback(t *testing.T) {
hash3, err := ledger.Commit(3)
assert.Nil(t, err)
assert.Equal(t, hash3, ledger.prevJournalHash)
assert.Equal(t, hash3, ledger.prevJnlHash)
err = ledger.Rollback(2)
assert.Nil(t, err)
assert.Equal(t, hash2, ledger.prevJournalHash)
assert.Equal(t, hash2, ledger.prevJnlHash)
assert.Equal(t, uint64(1), ledger.minJnlHeight)
assert.Equal(t, uint64(2), ledger.maxJnlHeight)
account0 := ledger.GetAccount(addr0)
assert.Equal(t, uint64(2), account0.GetBalance())
@ -161,9 +168,15 @@ func TestChainLedger_Rollback(t *testing.T) {
assert.Nil(t, account1.CodeHash())
assert.Nil(t, account1.Code())
ledger.Close()
ledger, err = New(repoRoot, blockStorage, log.NewWithModule("executor"))
assert.Nil(t, err)
assert.Equal(t, uint64(1), ledger.minJnlHeight)
assert.Equal(t, uint64(2), ledger.maxJnlHeight)
err = ledger.Rollback(1)
assert.Nil(t, err)
assert.Equal(t, hash1, ledger.prevJournalHash)
assert.Equal(t, hash1, ledger.prevJnlHash)
account0 = ledger.GetAccount(addr0)
assert.Equal(t, uint64(1), account0.GetBalance())
@ -172,4 +185,80 @@ func TestChainLedger_Rollback(t *testing.T) {
assert.Equal(t, code[:], account0.Code())
ok, _ = account0.GetState([]byte("a"))
assert.False(t, ok)
err = ledger.Rollback(0)
assert.Equal(t, ErrorRollbackTooMuch, err)
}
func TestChainLedger_RemoveJournalsBeforeBlock(t *testing.T) {
repoRoot, err := ioutil.TempDir("", "ledger_removeJournal")
assert.Nil(t, err)
blockStorage, err := leveldb.New(repoRoot)
assert.Nil(t, err)
ledger, err := New(repoRoot, blockStorage, log.NewWithModule("executor"))
assert.Nil(t, err)
assert.Equal(t, uint64(0), ledger.minJnlHeight)
assert.Equal(t, uint64(0), ledger.maxJnlHeight)
_, _ = ledger.Commit(1)
_, _ = ledger.Commit(2)
_, _ = ledger.Commit(3)
hash, _ := ledger.Commit(4)
assert.Equal(t, uint64(1), ledger.minJnlHeight)
assert.Equal(t, uint64(4), ledger.maxJnlHeight)
minHeight, maxHeight := getJournalRange(ledger.ldb)
journal := getBlockJournal(maxHeight, ledger.ldb)
assert.Equal(t, uint64(1), minHeight)
assert.Equal(t, uint64(4), maxHeight)
assert.Equal(t, hash, journal.ChangedHash)
err = ledger.RemoveJournalsBeforeBlock(5)
assert.Equal(t, ErrorRemoveJournalOutOfRange, err)
err = ledger.RemoveJournalsBeforeBlock(2)
assert.Nil(t, err)
assert.Equal(t, uint64(2), ledger.minJnlHeight)
assert.Equal(t, uint64(4), ledger.maxJnlHeight)
minHeight, maxHeight = getJournalRange(ledger.ldb)
journal = getBlockJournal(maxHeight, ledger.ldb)
assert.Equal(t, uint64(2), minHeight)
assert.Equal(t, uint64(4), maxHeight)
assert.Equal(t, hash, journal.ChangedHash)
err = ledger.RemoveJournalsBeforeBlock(2)
assert.Nil(t, err)
assert.Equal(t, uint64(2), ledger.minJnlHeight)
assert.Equal(t, uint64(4), ledger.maxJnlHeight)
err = ledger.RemoveJournalsBeforeBlock(1)
assert.Nil(t, err)
assert.Equal(t, uint64(2), ledger.minJnlHeight)
assert.Equal(t, uint64(4), ledger.maxJnlHeight)
err = ledger.RemoveJournalsBeforeBlock(4)
assert.Nil(t, err)
assert.Equal(t, uint64(4), ledger.minJnlHeight)
assert.Equal(t, uint64(4), ledger.maxJnlHeight)
assert.Equal(t, hash, ledger.prevJnlHash)
minHeight, maxHeight = getJournalRange(ledger.ldb)
journal = getBlockJournal(maxHeight, ledger.ldb)
assert.Equal(t, uint64(4), minHeight)
assert.Equal(t, uint64(4), maxHeight)
assert.Equal(t, hash, journal.ChangedHash)
ledger.Close()
ledger, err = New(repoRoot, blockStorage, log.NewWithModule("executor"))
assert.Nil(t, err)
assert.Equal(t, uint64(4), ledger.minJnlHeight)
assert.Equal(t, uint64(4), ledger.maxJnlHeight)
assert.Equal(t, hash, ledger.prevJnlHash)
}

14
internal/ledger/state_accessor.go
View File

@ -119,7 +119,7 @@ func (l *ChainLedger) Commit(height uint64) (types.Hash, error) {
for _, addr := range sortedAddr {
dirtyAccountData = append(dirtyAccountData, accountData[addr]...)
}
dirtyAccountData = append(dirtyAccountData, l.prevJournalHash[:]...)
dirtyAccountData = append(dirtyAccountData, l.prevJnlHash[:]...)
journalHash := sha256.Sum256(dirtyAccountData)
blockJournal := BlockJournal{
@ -133,11 +133,17 @@ func (l *ChainLedger) Commit(height uint64) (types.Hash, error) {
}
ldbBatch.Put(compositeKey(journalKey, height), data)
ldbBatch.Put(compositeKey(journalKey, maxHeightStr), marshalHeight(height))
if l.minJnlHeight == 0 {
l.minJnlHeight = height
ldbBatch.Put(compositeKey(journalKey, minHeightStr), marshalHeight(height))
}
ldbBatch.Commit()
l.height = height
l.prevJournalHash = journalHash
l.maxJnlHeight = height
l.prevJnlHash = journalHash
l.Clear()
return journalHash, nil
@ -145,5 +151,5 @@ func (l *ChainLedger) Commit(height uint64) (types.Hash, error) {
// Version returns the current version
func (l *ChainLedger) Version() uint64 {
return l.height
return l.maxJnlHeight
}

3
internal/ledger/types.go
View File

@ -19,6 +19,9 @@ type Ledger interface {
// Rollback
Rollback(height uint64) error
// RemoveJournalsBeforeBlock
RemoveJournalsBeforeBlock(height uint64) error
// Close release resource
Close()
}