diff --git a/action/protocol/staking/read_state.go b/action/protocol/staking/read_state.go index b72c56be55..fa2a0f5250 100644 --- a/action/protocol/staking/read_state.go +++ b/action/protocol/staking/read_state.go @@ -16,6 +16,10 @@ import ( "github.com/iotexproject/iotex-core/v2/state" ) +func ToIoTeXTypesVoteBucketList(sr protocol.StateReader, buckets []*VoteBucket) (*iotextypes.VoteBucketList, error) { + return toIoTeXTypesVoteBucketList(sr, buckets) +} + func toIoTeXTypesVoteBucketList(sr protocol.StateReader, buckets []*VoteBucket) (*iotextypes.VoteBucketList, error) { esr := NewEndorsementStateReader(sr) res := iotextypes.VoteBucketList{ @@ -98,6 +102,10 @@ func toIoTeXTypesCandidateV2(csr CandidateStateReader, cand *Candidate, featureC return c, nil } +func ToIoTeXTypesCandidateListV2(csr CandidateStateReader, candidates CandidateList, featureCtx protocol.FeatureCtx) (*iotextypes.CandidateListV2, error) { + return toIoTeXTypesCandidateListV2(csr, candidates, featureCtx) +} + func toIoTeXTypesCandidateListV2(csr CandidateStateReader, candidates CandidateList, featureCtx protocol.FeatureCtx) (*iotextypes.CandidateListV2, error) { res := iotextypes.CandidateListV2{ Candidates: make([]*iotextypes.CandidateV2, 0, len(candidates)), diff --git a/blockindex/nativestaking/bucket_list.go b/blockindex/nativestaking/bucket_list.go new file mode 100644 index 0000000000..c360ee361a --- /dev/null +++ b/blockindex/nativestaking/bucket_list.go @@ -0,0 +1,71 @@ +// Copyright (c) 2025 IoTeX Foundation +// This source code is provided 'as is' and no warranties are given as to title or non-infringement, merchantability +// or fitness for purpose and, to the extent permitted by law, all liability for your use of the code is disclaimed. +// This source code is governed by Apache License 2.0 that can be found in the LICENSE file. + +package stakingindex + +import ( + "google.golang.org/protobuf/proto" + + "github.com/iotexproject/iotex-core/v2/blockindex/nativestaking/stakingindexpb" +) + +type bucketList struct { + maxBucket uint64 + deleted []uint64 +} + +func (bl *bucketList) serialize() ([]byte, error) { + return proto.Marshal(bl.toProto()) +} + +func (bl *bucketList) toProto() *stakingindexpb.BucketList { + return &stakingindexpb.BucketList{ + MaxBucket: bl.maxBucket, + Deleted: bl.deleted, + } +} + +func fromProtoBucketList(pb *stakingindexpb.BucketList) *bucketList { + return &bucketList{ + maxBucket: pb.MaxBucket, + deleted: pb.Deleted, + } +} + +func deserializeBucketList(buf []byte) (*bucketList, error) { + pb := stakingindexpb.BucketList{} + if err := proto.Unmarshal(buf, &pb); err != nil { + return nil, err + } + return fromProtoBucketList(&pb), nil +} + +type candList struct { + id [][]byte +} + +func (cl *candList) serialize() ([]byte, error) { + return proto.Marshal(cl.toProto()) +} + +func (cl *candList) toProto() *stakingindexpb.CandList { + return &stakingindexpb.CandList{ + Id: cl.id, + } +} + +func fromProtoCandList(pb *stakingindexpb.CandList) *candList { + return &candList{ + id: pb.Id, + } +} + +func deserializeCandList(buf []byte) (*candList, error) { + pb := stakingindexpb.CandList{} + if err := proto.Unmarshal(buf, &pb); err != nil { + return nil, err + } + return fromProtoCandList(&pb), nil +} diff --git a/blockindex/nativestaking/bucket_list_test.go b/blockindex/nativestaking/bucket_list_test.go new file mode 100644 index 0000000000..9237b6f102 --- /dev/null +++ b/blockindex/nativestaking/bucket_list_test.go @@ -0,0 +1,65 @@ +// Copyright (c) 2025 IoTeX Foundation +// This source code is provided 'as is' and no warranties are given as to title or non-infringement, merchantability +// or fitness for purpose and, to the extent permitted by law, all liability for your use of the code is disclaimed. +// This source code is governed by Apache License 2.0 that can be found in the LICENSE file. + +package stakingindex + +import ( + ra "crypto/rand" + "math/rand/v2" + "testing" + + "github.com/stretchr/testify/require" + + . "github.com/iotexproject/iotex-core/v2/pkg/util/assertions" +) + +func TestBucketList(t *testing.T) { + req := require.New(t) + l0 := bucketList{0, nil} + b := MustNoErrorV(l0.serialize()) + l1 := MustNoErrorV(deserializeBucketList(b)) + req.Equal(&l0, l1) +} + +func TestBucketListSize(t *testing.T) { + req := require.New(t) + d, check := []uint64{}, []uint64{} + for i := range 50001 { + d = append(d, rand.Uint64N(50001)) + if i%5000 == 0 { + check = append(check, d[i]) + } + } + l0 := bucketList{d[1234], d} + b := MustNoErrorV(l0.serialize()) + l1 := MustNoErrorV(deserializeBucketList(b)) + req.Equal(&l0, l1) + req.Equal(d[1234], l1.maxBucket) + for i := 0; i < 50001; i += 5000 { + req.Equal(check[i/5000], l1.deleted[i]) + } +} + +func TestCandList(t *testing.T) { + req := require.New(t) + var ( + check [][]byte + l0 = candList{} + ) + for range 10 { + var b [20]byte + n := MustNoErrorV(ra.Read(b[:])) + req.Equal(20, n) + l0.id = append(l0.id, b[:]) + check = append(check, b[:]) + } + b := MustNoErrorV(l0.serialize()) + l1 := MustNoErrorV(deserializeCandList(b)) + req.Equal(10, len(l1.id)) + req.Equal(&l0, l1) + for i := range 10 { + req.Equal(check[i], l1.id[i]) + } +} diff --git a/blockindex/nativestaking/candBucketIndexer.go b/blockindex/nativestaking/candBucketIndexer.go new file mode 100644 index 0000000000..68c5781f1f --- /dev/null +++ b/blockindex/nativestaking/candBucketIndexer.go @@ -0,0 +1,335 @@ +// Copyright (c) 2025 IoTeX Foundation +// This source code is provided 'as is' and no warranties are given as to title or non-infringement, merchantability +// or fitness for purpose and, to the extent permitted by law, all liability for your use of the code is disclaimed. +// This source code is governed by Apache License 2.0 that can be found in the LICENSE file. + +package stakingindex + +import ( + "context" + "fmt" + "math" + + "github.com/pkg/errors" + "google.golang.org/protobuf/proto" + + "github.com/iotexproject/iotex-address/address" + "github.com/iotexproject/iotex-proto/golang/iotextypes" + + "github.com/iotexproject/iotex-core/v2/action/protocol" + "github.com/iotexproject/iotex-core/v2/action/protocol/staking" + "github.com/iotexproject/iotex-core/v2/blockchain/block" + "github.com/iotexproject/iotex-core/v2/db" + "github.com/iotexproject/iotex-core/v2/db/batch" + "github.com/iotexproject/iotex-core/v2/pkg/util/byteutil" +) + +const ( + // StakingCandidatesNamespace is a namespace to store candidates with epoch start height + StakingCandidatesNamespace = "stakingCandidates" + // StakingBucketsNamespace is a namespace to store vote buckets with epoch start height + StakingBucketsNamespace = "stakingBuckets" + // AccountKVNamespace is the bucket name for account + AccountKVNamespace = "Account" +) + +var ( + _currHeightKey = []byte("crh") + _currDeleteListKey = []byte("cdl") + _maxDeleteListKey = byteutil.Uint64ToBytesBigEndian(math.MaxUint64) + _maxCandListKey = []byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255} +) + +// CandBucketsIndexer is an indexer to store buckets and candidates by given height +type CandBucketsIndexer struct { + kvBase db.KVStore + kvVersioned db.KvVersioned + stateReader protocol.StateReader + currentHeight uint64 + deleteList *bucketList + candList *candList + candMap map[string]bool +} + +// NewCandBucketsIndexer creates a new indexer +func NewCandBucketsIndexer(kv db.KvVersioned) (*CandBucketsIndexer, error) { + if kv == nil { + return nil, errors.New("kvStore is nil") + } + return &CandBucketsIndexer{ + kvBase: kv.Base(), + kvVersioned: kv, + stateReader: newSRFromKVStore(kv.Base()), + candMap: map[string]bool{}, + }, nil +} + +// Start starts the indexer +func (cbi *CandBucketsIndexer) Start(ctx context.Context) error { + if err := cbi.kvVersioned.Start(ctx); err != nil { + return err + } + ret, err := cbi.kvBase.Get(db.MetadataNamespace, _currHeightKey) + switch errors.Cause(err) { + case nil: + cbi.currentHeight = byteutil.BytesToUint64BigEndian(ret) + case db.ErrNotExist: + cbi.currentHeight = 0 + default: + return err + } + if err := cbi.getDeleteList(); err != nil { + return err + } + if err := cbi.getCandList(); err != nil { + return err + } + // create cand map + for _, v := range cbi.candList.id { + cbi.candMap[string(v)] = true + } + return nil +} + +func (cbi *CandBucketsIndexer) getDeleteList() error { + ret, err := cbi.kvVersioned.SetVersion(cbi.currentHeight).Get(StakingBucketsNamespace, _maxDeleteListKey) + switch errors.Cause(err) { + case nil: + cbi.deleteList, err = deserializeBucketList(ret) + if err != nil { + return err + } + case db.ErrNotExist: + cbi.deleteList = &bucketList{} + default: + return err + } + return nil +} + +func (cbi *CandBucketsIndexer) getCandList() error { + ret, err := cbi.kvVersioned.SetVersion(cbi.currentHeight).Get(StakingCandidatesNamespace, _maxCandListKey) + switch errors.Cause(err) { + case nil: + cbi.candList, err = deserializeCandList(ret) + if err != nil { + return err + } + case db.ErrNotExist: + cbi.candList = &candList{} + default: + return err + } + return nil +} + +// Stop stops the indexer +func (cbi *CandBucketsIndexer) Stop(ctx context.Context) error { + return cbi.kvVersioned.Stop(ctx) +} + +func (cbi *CandBucketsIndexer) candBucketFromBlock(blk *block.Block) (staking.CandidateList, []*staking.VoteBucket, []uint64, error) { + // TODO: extract affected buckets and candidates from tx in block + var ( + b []*staking.VoteBucket + cl staking.CandidateList + d []uint64 + ) + if blk.Height() == 1 { + b = b1 + cl = c[:1] + } else if blk.Height() == 2 { + b = b2 + cl = c[1:2] + d = []uint64{1} + } else if blk.Height() == 3 { + b = b3 + cl = c[2:3] + d = []uint64{3, 4} + } + return cl, b, d, nil +} + +func (cbi *CandBucketsIndexer) PutBlock(ctx context.Context, blk *block.Block) error { + cands, changedBuckets, deletedBuckets, err := cbi.candBucketFromBlock(blk) + if err != nil { + return err + } + csr, err := staking.ConstructBaseView(cbi.stateReader) + if err != nil { + return err + } + candidateList, err := staking.ToIoTeXTypesCandidateListV2(csr, cands, protocol.MustGetFeatureCtx(ctx)) + if err != nil { + return err + } + bucketList, err := staking.ToIoTeXTypesVoteBucketList(cbi.stateReader, changedBuckets) + if err != nil { + return err + } + var ( + b = batch.NewBatch() + newCand bool + ) + for _, c := range candidateList.Candidates { + addr, err := address.FromString(c.Id) + if err != nil { + return err + } + cand, err := proto.Marshal(c) + if err != nil { + return err + } + ab := addr.Bytes() + b.Put(StakingCandidatesNamespace, ab, cand, fmt.Sprintf("failed to write cand = %x\n", cand)) + // update cand map/list + if as := string(ab); !cbi.candMap[as] { + cbi.candMap[as] = true + newCand = true + cbi.candList.id = append(cbi.candList.id, ab) + } + } + if newCand { + cand, err := cbi.candList.serialize() + if err != nil { + return err + } + b.Put(StakingCandidatesNamespace, _maxCandListKey, cand, fmt.Sprintf("failed to write cand list = %x\n", cand)) + } + for _, bucket := range bucketList.Buckets { + cb, err := proto.Marshal(bucket) + if err != nil { + return err + } + b.Put(StakingBucketsNamespace, byteutil.Uint64ToBytesBigEndian(bucket.Index), cb, fmt.Sprintf("failed to write bucket = %x\n", cb)) + } + // update deleted bucket list + var ( + newBucket uint64 + h = blk.Height() + ) + for _, v := range changedBuckets { + if v.Index > cbi.deleteList.maxBucket { + newBucket = v.Index + } + } + if newBucket > 0 || len(deletedBuckets) > 0 { + if newBucket > 0 { + cbi.deleteList.maxBucket = newBucket + } + if len(deletedBuckets) > 0 { + cbi.deleteList.deleted = append(cbi.deleteList.deleted, deletedBuckets...) + } + buf, err := cbi.deleteList.serialize() + if err != nil { + return err + } + b.Put(StakingBucketsNamespace, _maxDeleteListKey, buf, fmt.Sprintf("failed to write deleted bucket list = %d\n", h)) + } + if b.Size() == 0 { + return cbi.kvBase.Put(db.MetadataNamespace, _currHeightKey, byteutil.Uint64ToBytesBigEndian(h)) + } + // update height + b.Put(db.MetadataNamespace, _currHeightKey, byteutil.Uint64ToBytesBigEndian(h), fmt.Sprintf("failed to write height = %d\n", h)) + if err = cbi.kvVersioned.SetVersion(h).WriteBatch(b); err != nil { + return err + } + cbi.currentHeight = h + return nil +} + +// GetBuckets gets vote buckets from indexer given epoch start height +func (cbi *CandBucketsIndexer) GetBuckets(height uint64, offset, limit uint32) (*iotextypes.VoteBucketList, uint64, error) { + if height > cbi.currentHeight { + height = cbi.currentHeight + } + // get the delete list + buckets := &iotextypes.VoteBucketList{} + kv := cbi.kvVersioned.SetVersion(height) + ret, err := kv.Get(StakingBucketsNamespace, _maxDeleteListKey) + if cause := errors.Cause(err); cause == db.ErrNotExist || cause == db.ErrBucketNotExist { + return buckets, height, nil + } + if err != nil { + return nil, height, err + } + dList, err := deserializeBucketList(ret) + if err != nil { + return nil, height, err + } + dBuckets := map[uint64]bool{} + for _, v := range dList.deleted { + dBuckets[v] = true + } + for i := range dList.maxBucket + 1 { + if dBuckets[i] { + continue + } + ret, err = kv.Get(StakingBucketsNamespace, byteutil.Uint64ToBytesBigEndian(i)) + if err != nil { + return nil, height, err + } + b := iotextypes.VoteBucket{} + if err = proto.Unmarshal(ret, &b); err != nil { + return nil, height, err + } + buckets.Buckets = append(buckets.Buckets, &b) + } + length := uint32(len(buckets.Buckets)) + if offset >= length { + return &iotextypes.VoteBucketList{}, height, nil + } + end := offset + limit + if end > uint32(len(buckets.Buckets)) { + end = uint32(len(buckets.Buckets)) + } + buckets.Buckets = buckets.Buckets[offset:end] + return buckets, height, nil +} + +// GetCandidates gets candidates from indexer given epoch start height +func (cbi *CandBucketsIndexer) GetCandidates(height uint64, offset, limit uint32) (*iotextypes.CandidateListV2, uint64, error) { + if height > cbi.currentHeight { + height = cbi.currentHeight + } + candidateList := &iotextypes.CandidateListV2{} + kv := cbi.kvVersioned.SetVersion(height) + ret, err := kv.Get(StakingCandidatesNamespace, _maxCandListKey) + if cause := errors.Cause(err); cause == db.ErrNotExist || cause == db.ErrBucketNotExist { + return candidateList, height, nil + } + if err != nil { + return nil, height, err + } + cList, err := deserializeCandList(ret) + if err != nil { + return nil, height, err + } + for _, v := range cList.id { + ret, err = kv.Get(StakingCandidatesNamespace, v) + if err != nil { + return nil, height, err + } + c := iotextypes.CandidateV2{} + if err = proto.Unmarshal(ret, &c); err != nil { + return nil, height, err + } + candidateList.Candidates = append(candidateList.Candidates, &c) + } + length := uint32(len(candidateList.Candidates)) + if offset >= length { + return &iotextypes.CandidateListV2{}, height, nil + } + end := offset + limit + if end > uint32(len(candidateList.Candidates)) { + end = uint32(len(candidateList.Candidates)) + } + candidateList.Candidates = candidateList.Candidates[offset:end] + // fill id if it's empty for backward compatibility + for i := range candidateList.Candidates { + if candidateList.Candidates[i].Id == "" { + candidateList.Candidates[i].Id = candidateList.Candidates[i].OwnerAddress + } + } + return candidateList, height, nil +} diff --git a/blockindex/nativestaking/candBucketIndexer_test.go b/blockindex/nativestaking/candBucketIndexer_test.go new file mode 100644 index 0000000000..4c2a53f2c8 --- /dev/null +++ b/blockindex/nativestaking/candBucketIndexer_test.go @@ -0,0 +1,62 @@ +// Copyright (c) 2025 IoTeX Foundation +// This source code is provided 'as is' and no warranties are given as to title or non-infringement, merchantability +// or fitness for purpose and, to the extent permitted by law, all liability for your use of the code is disclaimed. +// This source code is governed by Apache License 2.0 that can be found in the LICENSE file. + +package stakingindex + +import ( + "context" + "testing" + + "github.com/iotexproject/go-pkgs/hash" + "github.com/stretchr/testify/require" + + "github.com/iotexproject/iotex-core/v2/action/protocol" + "github.com/iotexproject/iotex-core/v2/action/protocol/staking" + "github.com/iotexproject/iotex-core/v2/blockchain/block" + "github.com/iotexproject/iotex-core/v2/blockchain/genesis" + "github.com/iotexproject/iotex-core/v2/db" + . "github.com/iotexproject/iotex-core/v2/pkg/util/assertions" + "github.com/iotexproject/iotex-core/v2/testutil" +) + +func TestCandBucketIndexer(t *testing.T) { + req := require.New(t) + + cbIndexPath := MustNoErrorV(testutil.PathOfTempFile("cbindex")) + defer testutil.CleanupPath(cbIndexPath) + cfg := db.DefaultConfig + cfg.DbPath = cbIndexPath + kv := db.NewKVStoreWithVersion(cfg, db.VersionedNamespaceOption( + db.Namespace{StakingCandidatesNamespace, 20}, db.Namespace{StakingBucketsNamespace, 8})) + + ci := MustNoErrorV(NewCandBucketsIndexer(kv)) + ctx := context.Background() + req.NoError(ci.Start(ctx)) + defer func() { req.NoError(ci.Stop(ctx)) }() + req.Zero(ci.currentHeight) + ctx = protocol.WithFeatureCtx(protocol.WithBlockCtx(genesis.WithGenesisContext(ctx, genesis.Default), protocol.BlockCtx{})) + req.NoError(ci.PutBlock(ctx, block.NewBlockDeprecated(0, 1, hash.ZeroHash256, testutil.TimestampNow(), nil, nil))) + req.NoError(ci.PutBlock(ctx, block.NewBlockDeprecated(0, 2, hash.ZeroHash256, testutil.TimestampNow(), nil, nil))) + req.NoError(ci.PutBlock(ctx, block.NewBlockDeprecated(0, 3, hash.ZeroHash256, testutil.TimestampNow(), nil, nil))) + b, _, err := ci.GetBuckets(3, 0, 8) + req.NoError(err) + req.Equal(6, len(b.Buckets)) + for i, index := range []uint64{0, 2, 5, 6, 7, 8} { + req.Equal(index, b.Buckets[i].Index) + } + req.NoError(ci.Stop(ctx)) + req.NoError(ci.Start(ctx)) + req.Equal(&bucketList{maxBucket: 8, deleted: []uint64{1, 3, 4}}, ci.deleteList) + csr := MustNoErrorV(staking.ConstructBaseView(ci.stateReader)) + cList := MustNoErrorV(staking.ToIoTeXTypesCandidateListV2(csr, c, protocol.MustGetFeatureCtx(ctx))) + for i := range 3 { + cand, _, err := ci.GetCandidates(uint64(i+1), 0, 4) + req.NoError(err) + req.Equal(i+1, len(cand.Candidates)) + for j := range i + 1 { + req.Equal(cList.Candidates[j].String(), cand.Candidates[j].String()) + } + } +} diff --git a/blockindex/nativestaking/sr.go b/blockindex/nativestaking/sr.go new file mode 100644 index 0000000000..5f0de3dd11 --- /dev/null +++ b/blockindex/nativestaking/sr.go @@ -0,0 +1,205 @@ +package stakingindex + +import ( + "math/big" + "time" + + "github.com/iotexproject/iotex-address/address" + "github.com/pkg/errors" + + "github.com/iotexproject/iotex-core/v2/action/protocol" + "github.com/iotexproject/iotex-core/v2/action/protocol/staking" + "github.com/iotexproject/iotex-core/v2/db" + . "github.com/iotexproject/iotex-core/v2/pkg/util/assertions" + "github.com/iotexproject/iotex-core/v2/state" +) + +var ( + b1 = []*staking.VoteBucket{ + &staking.VoteBucket{ + Index: 0, + Candidate: &address.AddrV1{}, + Owner: &address.AddrV1{}, + StakedAmount: &big.Int{}, + StakedDuration: time.Duration(0), + CreateTime: time.Time{}, + StakeStartTime: time.Time{}, + UnstakeStartTime: time.Time{}, + }, + &staking.VoteBucket{ + Index: 1, + Candidate: &address.AddrV1{}, + Owner: &address.AddrV1{}, + StakedAmount: &big.Int{}, + StakedDuration: time.Duration(1), + CreateTime: time.Time{}, + StakeStartTime: time.Time{}, + UnstakeStartTime: time.Time{}, + }, + } + b2 = []*staking.VoteBucket{ + &staking.VoteBucket{ + Index: 2, + Candidate: &address.AddrV1{}, + Owner: &address.AddrV1{}, + StakedAmount: &big.Int{}, + StakedDuration: time.Duration(2), + CreateTime: time.Time{}, + StakeStartTime: time.Time{}, + UnstakeStartTime: time.Time{}, + }, + &staking.VoteBucket{ + Index: 3, + Candidate: &address.AddrV1{}, + Owner: &address.AddrV1{}, + StakedAmount: &big.Int{}, + StakedDuration: time.Duration(3), + CreateTime: time.Time{}, + StakeStartTime: time.Time{}, + UnstakeStartTime: time.Time{}, + }, + &staking.VoteBucket{ + Index: 4, + Candidate: &address.AddrV1{}, + Owner: &address.AddrV1{}, + StakedAmount: &big.Int{}, + StakedDuration: time.Duration(4), + CreateTime: time.Time{}, + StakeStartTime: time.Time{}, + UnstakeStartTime: time.Time{}, + }, + } + b3 = []*staking.VoteBucket{ + &staking.VoteBucket{ + Index: 5, + Candidate: &address.AddrV1{}, + Owner: &address.AddrV1{}, + StakedAmount: &big.Int{}, + StakedDuration: time.Duration(5), + CreateTime: time.Time{}, + StakeStartTime: time.Time{}, + UnstakeStartTime: time.Time{}, + }, + &staking.VoteBucket{ + Index: 6, + Candidate: &address.AddrV1{}, + Owner: &address.AddrV1{}, + StakedAmount: &big.Int{}, + StakedDuration: time.Duration(6), + CreateTime: time.Time{}, + StakeStartTime: time.Time{}, + UnstakeStartTime: time.Time{}, + }, + &staking.VoteBucket{ + Index: 7, + Candidate: &address.AddrV1{}, + Owner: &address.AddrV1{}, + StakedAmount: &big.Int{}, + StakedDuration: time.Duration(7), + CreateTime: time.Time{}, + StakeStartTime: time.Time{}, + UnstakeStartTime: time.Time{}, + }, + &staking.VoteBucket{ + Index: 8, + Candidate: &address.AddrV1{}, + Owner: &address.AddrV1{}, + StakedAmount: &big.Int{}, + StakedDuration: time.Duration(8), + CreateTime: time.Time{}, + StakeStartTime: time.Time{}, + UnstakeStartTime: time.Time{}, + }, + } + c = staking.CandidateList{ + &staking.Candidate{ + Owner: MustNoErrorV(address.FromBytes([]byte{1})), + Operator: MustNoErrorV(address.FromBytes([]byte{1})), + Reward: MustNoErrorV(address.FromBytes([]byte{1})), + Name: "1", + Votes: &big.Int{}, + SelfStake: &big.Int{}, + }, + &staking.Candidate{ + Owner: MustNoErrorV(address.FromBytes([]byte{2})), + Operator: MustNoErrorV(address.FromBytes([]byte{2})), + Reward: MustNoErrorV(address.FromBytes([]byte{2})), + Identifier: MustNoErrorV(address.FromBytes([]byte{2})), + Name: "2", + Votes: &big.Int{}, + SelfStake: &big.Int{}, + }, + &staking.Candidate{ + Owner: MustNoErrorV(address.FromBytes([]byte{3})), + Operator: MustNoErrorV(address.FromBytes([]byte{3})), + Reward: MustNoErrorV(address.FromBytes([]byte{3})), + Identifier: MustNoErrorV(address.FromBytes([]byte{3})), + Name: "3", + Votes: &big.Int{}, + SelfStake: &big.Int{}, + }, + } +) + +type srKV struct { + kv db.KVStore +} + +func newSRFromKVStore(kv db.KVStore) protocol.StateReader { + return &srKV{kv} +} + +func (sr *srKV) Height() (uint64, error) { + return 0, nil +} + +func (sr *srKV) State(s any, opts ...protocol.StateOption) (uint64, error) { + cfg, err := protocol.CreateStateConfig(opts...) + if err != nil { + return 0, err + } + if len(cfg.Namespace) == 0 { + cfg.Namespace = AccountKVNamespace + } + if cfg.Keys != nil { + return 0, errors.Wrap(errors.New("not supported"), "Read state with keys option has not been implemented yet") + } + + data, err := sr.kv.Get(cfg.Namespace, cfg.Key) + if err != nil { + if errors.Cause(err) == db.ErrNotExist { + return 0, errors.Wrapf(state.ErrStateNotExist, "state of %x doesn't exist", cfg.Key) + } + return 0, errors.Wrapf(err, "error when getting the state of %x", cfg.Key) + } + if err := state.Deserialize(s, data); err != nil { + return 0, errors.Wrapf(err, "error when deserializing state data into %T", s) + } + return 0, nil +} + +func (sr *srKV) States(opts ...protocol.StateOption) (uint64, state.Iterator, error) { + cfg, err := protocol.CreateStateConfig(opts...) + if err != nil { + return 0, nil, err + } + if len(cfg.Namespace) == 0 { + cfg.Namespace = AccountKVNamespace + } + if cfg.Keys != nil { + return 0, nil, errors.Wrap(errors.New("not supported"), "Read states with key option has not been implemented yet") + } + keys, values, err := db.ReadStates(sr.kv, cfg.Namespace, cfg.Keys) + if err != nil { + return 0, nil, err + } + iter, err := state.NewIterator(keys, values) + if err != nil { + return 0, nil, err + } + return 0, iter, nil +} + +func (sr *srKV) ReadView(string) (protocol.View, error) { + return &staking.ViewData{}, nil +} diff --git a/blockindex/nativestaking/stakingindexpb/staking.pb.go b/blockindex/nativestaking/stakingindexpb/staking.pb.go new file mode 100644 index 0000000000..2e4f1e8f30 --- /dev/null +++ b/blockindex/nativestaking/stakingindexpb/staking.pb.go @@ -0,0 +1,187 @@ +// Copyright (c) 2025 IoTeX +// This source code is provided 'as is' and no warranties are given as to title or non-infringement, merchantability +// or fitness for purpose and, to the extent permitted by law, all liability for your use of the code is disclaimed. +// This source code is governed by Apache License 2.0 that can be found in the LICENSE file. + +// To compile the proto, run: +// protoc --go_out=. --go-grpc_out=. *.proto + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.6 +// protoc v5.29.3 +// source: blockindex/nativestaking/stakingindexpb/staking.proto + +package stakingindexpb + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type BucketList struct { + state protoimpl.MessageState `protogen:"open.v1"` + MaxBucket uint64 `protobuf:"varint,1,opt,name=maxBucket,proto3" json:"maxBucket,omitempty"` + Deleted []uint64 `protobuf:"varint,2,rep,packed,name=deleted,proto3" json:"deleted,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BucketList) Reset() { + *x = BucketList{} + mi := &file_blockindex_nativestaking_stakingindexpb_staking_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BucketList) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BucketList) ProtoMessage() {} + +func (x *BucketList) ProtoReflect() protoreflect.Message { + mi := &file_blockindex_nativestaking_stakingindexpb_staking_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BucketList.ProtoReflect.Descriptor instead. +func (*BucketList) Descriptor() ([]byte, []int) { + return file_blockindex_nativestaking_stakingindexpb_staking_proto_rawDescGZIP(), []int{0} +} + +func (x *BucketList) GetMaxBucket() uint64 { + if x != nil { + return x.MaxBucket + } + return 0 +} + +func (x *BucketList) GetDeleted() []uint64 { + if x != nil { + return x.Deleted + } + return nil +} + +type CandList struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id [][]byte `protobuf:"bytes,1,rep,name=id,proto3" json:"id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CandList) Reset() { + *x = CandList{} + mi := &file_blockindex_nativestaking_stakingindexpb_staking_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CandList) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CandList) ProtoMessage() {} + +func (x *CandList) ProtoReflect() protoreflect.Message { + mi := &file_blockindex_nativestaking_stakingindexpb_staking_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CandList.ProtoReflect.Descriptor instead. +func (*CandList) Descriptor() ([]byte, []int) { + return file_blockindex_nativestaking_stakingindexpb_staking_proto_rawDescGZIP(), []int{1} +} + +func (x *CandList) GetId() [][]byte { + if x != nil { + return x.Id + } + return nil +} + +var File_blockindex_nativestaking_stakingindexpb_staking_proto protoreflect.FileDescriptor + +const file_blockindex_nativestaking_stakingindexpb_staking_proto_rawDesc = "" + + "\n" + + "5blockindex/nativestaking/stakingindexpb/staking.proto\x12\x0estakingindexpb\"D\n" + + "\n" + + "bucketList\x12\x1c\n" + + "\tmaxBucket\x18\x01 \x01(\x04R\tmaxBucket\x12\x18\n" + + "\adeleted\x18\x02 \x03(\x04R\adeleted\"\x1a\n" + + "\bcandList\x12\x0e\n" + + "\x02id\x18\x01 \x03(\fR\x02idBOZMgithub.com/iotexproject/iotex-core/v2/blockindex/nativestaking/stakingindexpbb\x06proto3" + +var ( + file_blockindex_nativestaking_stakingindexpb_staking_proto_rawDescOnce sync.Once + file_blockindex_nativestaking_stakingindexpb_staking_proto_rawDescData []byte +) + +func file_blockindex_nativestaking_stakingindexpb_staking_proto_rawDescGZIP() []byte { + file_blockindex_nativestaking_stakingindexpb_staking_proto_rawDescOnce.Do(func() { + file_blockindex_nativestaking_stakingindexpb_staking_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_blockindex_nativestaking_stakingindexpb_staking_proto_rawDesc), len(file_blockindex_nativestaking_stakingindexpb_staking_proto_rawDesc))) + }) + return file_blockindex_nativestaking_stakingindexpb_staking_proto_rawDescData +} + +var file_blockindex_nativestaking_stakingindexpb_staking_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_blockindex_nativestaking_stakingindexpb_staking_proto_goTypes = []any{ + (*BucketList)(nil), // 0: stakingindexpb.bucketList + (*CandList)(nil), // 1: stakingindexpb.candList +} +var file_blockindex_nativestaking_stakingindexpb_staking_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_blockindex_nativestaking_stakingindexpb_staking_proto_init() } +func file_blockindex_nativestaking_stakingindexpb_staking_proto_init() { + if File_blockindex_nativestaking_stakingindexpb_staking_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_blockindex_nativestaking_stakingindexpb_staking_proto_rawDesc), len(file_blockindex_nativestaking_stakingindexpb_staking_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_blockindex_nativestaking_stakingindexpb_staking_proto_goTypes, + DependencyIndexes: file_blockindex_nativestaking_stakingindexpb_staking_proto_depIdxs, + MessageInfos: file_blockindex_nativestaking_stakingindexpb_staking_proto_msgTypes, + }.Build() + File_blockindex_nativestaking_stakingindexpb_staking_proto = out.File + file_blockindex_nativestaking_stakingindexpb_staking_proto_goTypes = nil + file_blockindex_nativestaking_stakingindexpb_staking_proto_depIdxs = nil +} diff --git a/blockindex/nativestaking/stakingindexpb/staking.proto b/blockindex/nativestaking/stakingindexpb/staking.proto new file mode 100644 index 0000000000..c4687398bc --- /dev/null +++ b/blockindex/nativestaking/stakingindexpb/staking.proto @@ -0,0 +1,19 @@ +// Copyright (c) 2025 IoTeX +// This source code is provided 'as is' and no warranties are given as to title or non-infringement, merchantability +// or fitness for purpose and, to the extent permitted by law, all liability for your use of the code is disclaimed. +// This source code is governed by Apache License 2.0 that can be found in the LICENSE file. + +// To compile the proto, run: +// protoc --go_out=. --go-grpc_out=. *.proto +syntax = "proto3"; +package stakingindexpb; +option go_package = "github.com/iotexproject/iotex-core/v2/blockindex/nativestaking/stakingindexpb"; + +message bucketList { + uint64 maxBucket = 1; + repeated uint64 deleted = 2; +} + +message candList { + repeated bytes id = 1; +} diff --git a/db/builder.go b/db/builder.go index c8c249d983..73f9ce0a57 100644 --- a/db/builder.go +++ b/db/builder.go @@ -5,6 +5,8 @@ import "github.com/pkg/errors" var ( // ErrEmptyDBPath is the error when db path is empty ErrEmptyDBPath = errors.New("empty db path") + // ErrEmptyVersionedNamespace is the error of empty versioned namespace + ErrEmptyVersionedNamespace = errors.New("cannot create versioned KVStore with empty versioned namespace") ) // CreateKVStore creates db from config and db path @@ -32,3 +34,20 @@ func CreateKVStoreWithCache(cfg Config, dbPath string, cacheSize int) (KVStore, return NewKvStoreWithCache(dao, cacheSize), nil } + +// CreateKVStoreVersioned creates versioned db from config and db path +func CreateKVStoreVersioned(cfg Config, dbPath string, vns []Namespace) (KVStore, error) { + if len(dbPath) == 0 { + return nil, ErrEmptyDBPath + } + if len(vns) == 0 { + return nil, ErrEmptyVersionedNamespace + } + for i := range vns { + if len(vns[i].Ns) == 0 || vns[i].KeyLen == 0 { + return nil, ErrEmptyVersionedNamespace + } + } + cfg.DbPath = dbPath + return NewKVStoreWithVersion(cfg, VersionedNamespaceOption(vns...)), nil +} diff --git a/db/db_versioned.go b/db/db_versioned.go index 2166ae1af2..b654ee62ca 100644 --- a/db/db_versioned.go +++ b/db/db_versioned.go @@ -32,6 +32,9 @@ type ( VersionedDB interface { lifecycle.StartStopper + // Base returns the underlying KVStore + Base() KVStore + // Put insert or update a record identified by (namespace, key) Put(uint64, string, []byte, []byte) error @@ -59,8 +62,8 @@ type ( // Namespace specifies the name and key length of the versioned namespace Namespace struct { - ns string - keyLen uint32 + Ns string + KeyLen uint32 } ) @@ -70,7 +73,7 @@ type BoltDBVersionedOption func(*BoltDBVersioned) func VnsOption(ns ...Namespace) BoltDBVersionedOption { return func(k *BoltDBVersioned) { for _, v := range ns { - k.vns[v.ns] = int(v.keyLen) + k.vns[v.Ns] = int(v.KeyLen) } } } @@ -122,6 +125,10 @@ func (b *BoltDBVersioned) addVersionedNamespace() error { return nil } +func (b *BoltDBVersioned) Base() KVStore { + return b.db +} + // Put writes a record func (b *BoltDBVersioned) Put(version uint64, ns string, key, value []byte) error { if !b.db.IsReady() { diff --git a/db/kvstore.go b/db/kvstore.go index 3607784dc4..d9a8eef3eb 100644 --- a/db/kvstore.go +++ b/db/kvstore.go @@ -6,9 +6,11 @@ package db import ( - "github.com/iotexproject/iotex-core/v2/pkg/lifecycle" + "github.com/pkg/errors" "github.com/iotexproject/iotex-core/v2/db/batch" + "github.com/iotexproject/iotex-core/v2/pkg/lifecycle" + "github.com/iotexproject/iotex-core/v2/state" ) type ( @@ -62,3 +64,34 @@ type ( SeekPrev([]byte, uint64) ([]byte, error) } ) + +func ReadStates(kvStore KVStore, namespace string, keys [][]byte) ([][]byte, [][]byte, error) { + var ( + ks, values [][]byte + err error + ) + if keys == nil { + ks, values, err = kvStore.Filter(namespace, func(k, v []byte) bool { return true }, nil, nil) + if err != nil { + if errors.Cause(err) == ErrNotExist || errors.Cause(err) == ErrBucketNotExist { + return nil, nil, errors.Wrapf(state.ErrStateNotExist, "failed to get states of ns = %x", namespace) + } + return nil, nil, err + } + return ks, values, nil + } + for _, key := range keys { + value, err := kvStore.Get(namespace, key) + switch errors.Cause(err) { + case ErrNotExist, ErrBucketNotExist: + values = append(values, nil) + ks = append(ks, key) + case nil: + values = append(values, value) + ks = append(ks, key) + default: + return nil, nil, err + } + } + return ks, values, nil +} diff --git a/db/kvstore_versioned.go b/db/kvstore_versioned.go index f15eb790ed..c7fa93b567 100644 --- a/db/kvstore_versioned.go +++ b/db/kvstore_versioned.go @@ -13,6 +13,11 @@ import ( "github.com/iotexproject/iotex-core/v2/pkg/lifecycle" ) +const ( + // MetadataNamespace is the default namespace to store metadata + MetadataNamespace = "metadata" +) + type ( // KvVersioned is a versioned key-value store, where each key has multiple // versions of value (corresponding to different heights in a blockchain) @@ -44,6 +49,9 @@ type ( KvVersioned interface { lifecycle.StartStopper + // Base returns the underlying KVStore + Base() KVStore + // Version returns the key's most recent version Version(string, []byte) (uint64, error) @@ -94,6 +102,10 @@ func (b *KvWithVersion) Stop(ctx context.Context) error { return b.db.Stop(ctx) } +func (b *KvWithVersion) Base() KVStore { + return b.db.Base() +} + // Put writes a record func (b *KvWithVersion) Put(ns string, key, value []byte) error { return b.db.Put(b.version, ns, key, value) diff --git a/state/factory/statedb.go b/state/factory/statedb.go index 4158a0d49a..a669cb1a10 100644 --- a/state/factory/statedb.go +++ b/state/factory/statedb.go @@ -393,7 +393,7 @@ func (sdb *stateDB) States(opts ...protocol.StateOption) (uint64, state.Iterator if cfg.Key != nil { return sdb.currentChainHeight, nil, errors.Wrap(ErrNotSupported, "Read states with key option has not been implemented yet") } - keys, values, err := readStates(sdb.dao.atHeight(sdb.currentChainHeight), cfg.Namespace, cfg.Keys) + keys, values, err := db.ReadStates(sdb.dao.atHeight(sdb.currentChainHeight), cfg.Namespace, cfg.Keys) if err != nil { return 0, nil, err } diff --git a/state/factory/util.go b/state/factory/util.go index 157e1b3c6c..18f32e07a7 100644 --- a/state/factory/util.go +++ b/state/factory/util.go @@ -19,7 +19,6 @@ import ( "github.com/iotexproject/iotex-core/v2/db" "github.com/iotexproject/iotex-core/v2/db/trie" "github.com/iotexproject/iotex-core/v2/db/trie/mptrie" - "github.com/iotexproject/iotex-core/v2/state" ) func processOptions(opts ...protocol.StateOption) (*protocol.StateConfig, error) { @@ -117,37 +116,6 @@ func protocolCommit(ctx context.Context, sr protocol.StateManager) error { return nil } -func readStates(kvStore db.KVStore, namespace string, keys [][]byte) ([][]byte, [][]byte, error) { - var ( - ks, values [][]byte - err error - ) - if keys == nil { - ks, values, err = kvStore.Filter(namespace, func(k, v []byte) bool { return true }, nil, nil) - if err != nil { - if errors.Cause(err) == db.ErrNotExist || errors.Cause(err) == db.ErrBucketNotExist { - return nil, nil, errors.Wrapf(state.ErrStateNotExist, "failed to get states of ns = %x", namespace) - } - return nil, nil, err - } - return ks, values, nil - } - for _, key := range keys { - value, err := kvStore.Get(namespace, key) - switch errors.Cause(err) { - case db.ErrNotExist, db.ErrBucketNotExist: - values = append(values, nil) - ks = append(ks, key) - case nil: - values = append(values, value) - ks = append(ks, key) - default: - return nil, nil, err - } - } - return ks, values, nil -} - func newTwoLayerTrie(ns string, dao db.KVStore, rootKey string, create bool) (trie.TwoLayerTrie, error) { dbForTrie, err := trie.NewKVStore(ns, dao) if err != nil { diff --git a/state/factory/workingsetstore.go b/state/factory/workingsetstore.go index 8e935a070d..16da2f9e3c 100644 --- a/state/factory/workingsetstore.go +++ b/state/factory/workingsetstore.go @@ -138,9 +138,9 @@ func (store *stateDBWorkingSetStore) Get(ns string, key []byte) ([]byte, error) func (store *stateDBWorkingSetStore) States(ns string, keys [][]byte) ([][]byte, [][]byte, error) { if store.readBuffer { // TODO: after the 180 HF, we can revert readBuffer, and always go this case - return readStates(store.flusher.KVStoreWithBuffer(), ns, keys) + return db.ReadStates(store.flusher.KVStoreWithBuffer(), ns, keys) } - return readStates(store.flusher.BaseKVStore(), ns, keys) + return db.ReadStates(store.flusher.BaseKVStore(), ns, keys) } func (store *stateDBWorkingSetStore) Finalize(height uint64) error {