diff --git a/beacon/storage.go b/beacon/storage.go index 6a6cf8b..0c61441 100644 --- a/beacon/storage.go +++ b/beacon/storage.go @@ -134,6 +134,10 @@ func (bs *Storage) Radius() *uint256.Int { return storage2.MaxDistance } +func (bs *Storage) Close() error { + return bs.db.Close() +} + func (bs *Storage) getContentValue(contentId []byte) ([]byte, error) { res := make([]byte, 0) err := bs.db.QueryRowContext(context.Background(), ContentValueLookupQueryBeacon, contentId).Scan(&res) diff --git a/cmd/shisui/main.go b/cmd/shisui/main.go index c884619..90bc010 100644 --- a/cmd/shisui/main.go +++ b/cmd/shisui/main.go @@ -39,6 +39,7 @@ import ( "github.com/optimism-java/shisui2/portalwire" "github.com/optimism-java/shisui2/state" "github.com/optimism-java/shisui2/storage" + "github.com/optimism-java/shisui2/storage/sqlite" "github.com/optimism-java/shisui2/web3" "github.com/protolambda/zrnt/eth2/configs" "github.com/urfave/cli/v2" @@ -378,11 +379,11 @@ func doPortMapping(natm nat.Interface, ln *enode.LocalNode, addr *net.UDPAddr) { func initHistory(config Config, server *rpc.Server, conn discover.UDPConn, localNode *enode.LocalNode, discV5 *discover.UDPv5, utp *portalwire.PortalUtp) (*history.Network, error) { networkName := portalwire.History.Name() - db, err := history.NewDB(config.DataDir, networkName) + db, err := sqlite.NewDB(config.DataDir, networkName) if err != nil { return nil, err } - contentStorage, err := history.NewHistoryStorage(storage.PortalStorageConfig{ + contentStorage, err := sqlite.NewHistoryStorage(storage.PortalStorageConfig{ StorageCapacityMB: config.DataCapacity, DB: db, NodeId: localNode.ID(), @@ -472,11 +473,11 @@ func initBeacon(config Config, server *rpc.Server, conn discover.UDPConn, localN func initState(config Config, server *rpc.Server, conn discover.UDPConn, localNode *enode.LocalNode, discV5 *discover.UDPv5, utp *portalwire.PortalUtp) (*state.Network, error) { networkName := portalwire.State.Name() - db, err := history.NewDB(config.DataDir, networkName) + db, err := sqlite.NewDB(config.DataDir, networkName) if err != nil { return nil, err } - contentStorage, err := history.NewHistoryStorage(storage.PortalStorageConfig{ + contentStorage, err := sqlite.NewHistoryStorage(storage.PortalStorageConfig{ StorageCapacityMB: config.DataCapacity, DB: db, NodeId: localNode.ID(), diff --git a/history/history.sqlite b/history/history.sqlite deleted file mode 100644 index 587022d..0000000 Binary files a/history/history.sqlite and /dev/null differ diff --git a/history/new_storage.go b/history/new_storage.go deleted file mode 100644 index 43c6d66..0000000 --- a/history/new_storage.go +++ /dev/null @@ -1,443 +0,0 @@ -package history - -// -// import ( -// "encoding/binary" -// "errors" -// "fmt" -// "path" -// "sync/atomic" -// -// "github.com/cockroachdb/pebble" -// "github.com/ethereum/go-ethereum/log" -// "github.com/ethereum/go-ethereum/metrics" -// "github.com/ethereum/go-ethereum/p2p/enode" -// "github.com/ethereum/go-ethereum/portalnetwork/storage" -// "github.com/holiman/uint256" -//) -// -// const ( -// contentDeletionFraction = 0.05 -// prefixContent = byte(0x01) // prefixContent + distance + contentId -> content -// prefixDistanceSize = byte(0x02) // prefixDistanceSize + distance -> total size -//) -// -//type ContentStorage struct { -// nodeId enode.ID -// storageCapacityInBytes uint64 -// radius atomic.Value -// db *pebble.DB -// log log.Logger -//} -// -//func NewHistoryStorage(config storage.PortalStorageConfig) (storage.ContentStorage, error) { -// dbPath := path.Join(config.DataDir, config.NetworkName) -// -// opts := &pebble.Options{ -// MaxOpenFiles: 1000, -// } -// -// db, err := pebble.Open(dbPath, opts) -// if err != nil { -// return nil, err -// } -// -// cs := &ContentStorage{ -// nodeId: config.NodeId, -// db: db, -// storageCapacityInBytes: config.StorageCapacityMB * 1000000, -// log: log.New("storage", config.NetworkName), -// } -// cs.radius.Store(storage.MaxDistance) -// cs.setRadiusToFarthestDistance() -// -// return cs, nil -//} -// -//func makeKey(prefix byte, distance []byte, contentId []byte) []byte { -// if contentId == nil { -// key := make([]byte, 1+len(distance)) -// key[0] = prefix -// copy(key[1:], distance) -// return key -// } -// key := make([]byte, 1+len(distance)+len(contentId)) -// key[0] = prefix -// copy(key[1:], distance) -// copy(key[1+len(distance):], contentId) -// return key -//} -// -//func (p *ContentStorage) Put(contentKey []byte, contentId []byte, content []byte) error { -// distance := xor(contentId, p.nodeId[:]) -// key := makeKey(prefixContent, distance, contentId) -// -// batch := p.db.NewBatch() -// defer batch.Close() -// -// // Update content -// if err := batch.Set(key, content, pebble.Sync); err != nil { -// return err -// } -// -// // Update distance size index -// sizeKey := makeKey(prefixDistanceSize, distance, nil) -// var currentSize uint64 -// if value, closer, err := p.db.Get(sizeKey); err == nil { -// currentSize = binary.BigEndian.Uint64(value) -// closer.Close() -// } -// -// newSize := currentSize + uint64(len(content)) -// sizeBytes := make([]byte, 8) -// binary.BigEndian.PutUint64(sizeBytes, newSize) -// -// if err := batch.Set(sizeKey, sizeBytes, pebble.Sync); err != nil { -// return err -// } -// -// if err := batch.Commit(pebble.Sync); err != nil { -// return err -// } -// -// if size, _ := p.UsedSize(); size > p.storageCapacityInBytes { -// if _, err := p.deleteContentFraction(contentDeletionFraction); err != nil { -// p.log.Warn("failed to delete oversize content", "err", err) -// } -// } -// -// if metrics.Enabled { -// portalStorageMetrics.EntriesCount.Inc(1) -// portalStorageMetrics.ContentStorageUsage.Inc(int64(len(content))) -// } -// -// return nil -//} -// -//func (p *ContentStorage) Get(contentKey []byte, contentId []byte) ([]byte, error) { -// distance := xor(contentId, p.nodeId[:]) -// key := makeKey(prefixContent, distance, contentId) -// -// value, closer, err := p.db.Get(key) -// if err == pebble.ErrNotFound { -// return nil, storage.ErrContentNotFound -// } -// if err != nil { -// return nil, err -// } -// defer closer.Close() -// -// return value, nil -//} -// -//func (p *ContentStorage) deleteContentFraction(fraction float64) (deleteCount int, err error) { -// if fraction <= 0 || fraction >= 1 { -// return 0, errors.New("fraction should be between 0 and 1") -// } -// -// totalSize, err := p.ContentSize() -// if err != nil { -// return 0, err -// } -// -// targetSize := uint64(float64(totalSize) * fraction) -// deletedSize := uint64(0) -// count := 0 -// -// iter := p.db.NewIter(&pebble.IterOptions{ -// LowerBound: []byte{prefixContent}, -// UpperBound: []byte{prefixContent + 1}, -// }) -// defer iter.Close() -// -// batch := p.db.NewBatch() -// defer batch.Close() -// -// for iter.Last(); iter.Valid() && deletedSize < targetSize; iter.Prev() { -// key := iter.Key() -// value := iter.Value() -// distance := key[1:33] -// -// // Delete content -// if err := batch.Delete(key, nil); err != nil { -// return count, err -// } -// -// // Update distance size index -// sizeKey := makeKey(prefixDistanceSize, distance, nil) -// var currentSize uint64 -// sizeValue, closer, err := p.db.Get(sizeKey) -// if err == nil { -// currentSize = binary.BigEndian.Uint64(sizeValue) -// closer.Close() -// } -// -// newSize := currentSize - uint64(len(value)) -// if newSize == 0 { -// if err := batch.Delete(sizeKey, nil); err != nil { -// return count, err -// } -// } else { -// sizeBytes := make([]byte, 8) -// binary.BigEndian.PutUint64(sizeBytes, newSize) -// if err := batch.Set(sizeKey, sizeBytes, nil); err != nil { -// return count, err -// } -// } -// -// deletedSize += uint64(len(value)) -// count++ -// -// if batch.Len() >= 1000 { -// if err := batch.Commit(pebble.Sync); err != nil { -// return count, err -// } -// batch = p.db.NewBatch() -// } -// } -// if batch.Len() > 0 { -// if err := batch.Commit(pebble.Sync); err != nil { -// return count, err -// } -// } -// -// if iter.Valid() { -// key := iter.Key() -// distance := key[1:33] -// dis := uint256.NewInt(0) -// if err := dis.UnmarshalSSZ(distance); err != nil { -// return count, err -// } -// p.radius.Store(dis) -// } -// -// return count, nil -//} -// -//func (p *ContentStorage) UsedSize() (uint64, error) { -// var totalSize uint64 -// iter := p.db.NewIter(&pebble.IterOptions{ -// LowerBound: []byte{prefixDistanceSize}, -// UpperBound: []byte{prefixDistanceSize + 1}, -// }) -// defer iter.Close() -// -// for iter.First(); iter.Valid(); iter.Next() { -// size := binary.BigEndian.Uint64(iter.Value()) -// totalSize += size -// } -// -// return totalSize, nil -//} -// -//func (p *ContentStorage) ContentSize() (uint64, error) { -// return p.UsedSize() -//} -// -//func (p *ContentStorage) ContentCount() (uint64, error) { -// var count uint64 -// iter, _ := p.db.NewIter(&pebble.IterOptions{ -// LowerBound: []byte{prefixContent}, -// UpperBound: []byte{prefixContent + 1}, -// }) -// defer iter.Close() -// -// for iter.First(); iter.Valid(); iter.Next() { -// count++ -// } -// -// return count, nil -//} -// -//func (p *ContentStorage) Radius() *uint256.Int { -// radius := p.radius.Load() -// val := radius.(*uint256.Int) -// return val -//} -//func (p *ContentStorage) GetLargestDistance() (*uint256.Int, error) { -// iter := p.db.NewIter(&pebble.IterOptions{ -// LowerBound: []byte{prefixContent}, -// UpperBound: []byte{prefixContent + 1}, -// }) -// defer iter.Close() -// -// if !iter.Last() { -// return nil, fmt.Errorf("no content found") -// } -// -// key := iter.Key() -// distance := key[1:33] -// -// res := uint256.NewInt(0) -// err := res.UnmarshalSSZ(distance) -// return res, err -//} -// -//func (p *ContentStorage) EstimateNewRadius(currentRadius *uint256.Int) (*uint256.Int, error) { -// currrentSize, err := p.UsedSize() -// if err != nil { -// return nil, err -// } -// -// sizeRatio := currrentSize / p.storageCapacityInBytes -// if sizeRatio > 0 { -// newRadius := new(uint256.Int).Div(currentRadius, uint256.NewInt(sizeRatio)) -// -// if metrics.Enabled { -// ratio := new(uint256.Int).Mul(newRadius, uint256.NewInt(100)) -// ratio.Mod(ratio, storage.MaxDistance) -// portalStorageMetrics.RadiusRatio.Update(ratio.Float64() / 100) -// } -// -// return newRadius, nil -// } -// return currentRadius, nil -//} -// -//func (p *ContentStorage) setRadiusToFarthestDistance() { -// largestDistance, err := p.GetLargestDistance() -// if err != nil { -// p.log.Error("failed to get farthest distance", "err", err) -// return -// } -// p.radius.Store(largestDistance) -//} -//func (p *ContentStorage) ForcePrune(radius *uint256.Int) error { -// batch := p.db.NewBatch() -// defer batch.Close() -// -// iter := p.db.NewIter(&pebble.IterOptions{ -// LowerBound: []byte{prefixContent}, -// UpperBound: []byte{prefixContent + 1}, -// }) -// defer iter.Close() -// -// var deletedSize int64 -// deleteCount := 0 -// -// for iter.First(); iter.Valid(); iter.Next() { -// key := iter.Key() -// value := iter.Value() -// distance := key[1:33] -// -// dis := uint256.NewInt(0) -// if err := dis.UnmarshalSSZ(distance); err != nil { -// return err -// } -// -// if dis.Cmp(radius) > 0 { -// // Delete content -// if err := batch.Delete(key, nil); err != nil { -// return err -// } -// -// // Update distance size index -// sizeKey := makeKey(prefixDistanceSize, distance, nil) -// var currentSize uint64 -// if sizeValue, closer, err := p.db.Get(sizeKey); err == nil { -// currentSize = binary.BigEndian.Uint64(sizeValue) -// closer.Close() -// } -// -// newSize := currentSize - uint64(len(value)) -// if newSize == 0 { -// if err := batch.Delete(sizeKey, nil); err != nil { -// return err -// } -// } else { -// sizeBytes := make([]byte, 8) -// binary.BigEndian.PutUint64(sizeBytes, newSize) -// if err := batch.Set(sizeKey, sizeBytes, nil); err != nil { -// return err -// } -// } -// -// deletedSize += int64(len(value)) -// deleteCount++ -// } -// -// if batch.Len() >= 1000 { -// if err := batch.Commit(pebble.Sync); err != nil { -// return err -// } -// batch = p.db.NewBatch() -// } -// } -// if batch.Len() > 0 { -// if err := batch.Commit(pebble.Sync); err != nil { -// return err -// } -// } -// -// if metrics.Enabled { -// portalStorageMetrics.EntriesCount.Dec(int64(deleteCount)) -// portalStorageMetrics.ContentStorageUsage.Dec(deletedSize) -// } -// -// return nil -//} -// -//func (p *ContentStorage) ReclaimSpace() error { -// return p.db.Compact([]byte{prefixContent}, []byte{prefixContent + 1}, true) -//} -// -//func (p *ContentStorage) Close() error { -// return p.db.Close() -//} -// -//func (p *ContentStorage) SizeByKey(contentId []byte) (uint64, error) { -// distance := xor(contentId, p.nodeId[:]) -// key := makeKey(prefixContent, distance, contentId) -// -// value, closer, err := p.db.Get(key) -// if err == pebble.ErrNotFound { -// return 0, nil -// } -// if err != nil { -// return 0, err -// } -// defer closer.Close() -// -// return uint64(len(value)), nil -//} -// -//func (p *ContentStorage) SizeByKeys(ids [][]byte) (uint64, error) { -// var totalSize uint64 -// -// for _, id := range ids { -// size, err := p.SizeByKey(id) -// if err != nil { -// return 0, err -// } -// totalSize += size -// } -// -// return totalSize, nil -//} -// -//func (p *ContentStorage) SizeOutRadius(radius *uint256.Int) (uint64, error) { -// var totalSize uint64 -// -// iter := p.db.NewIter(&pebble.IterOptions{ -// LowerBound: []byte{prefixDistanceSize}, -// UpperBound: []byte{prefixDistanceSize + 1}, -// }) -// defer iter.Close() -// -// for iter.First(); iter.Valid(); iter.Next() { -// key := iter.Key() -// distance := key[1:33] -// -// dis := uint256.NewInt(0) -// if err := dis.UnmarshalSSZ(distance); err != nil { -// return 0, err -// } -// -// if dis.Cmp(radius) > 0 { -// size := binary.BigEndian.Uint64(iter.Value()) -// totalSize += size -// } -// } -// -// return totalSize, nil -//} diff --git a/state/storage.go b/state/storage.go index 6c22eb5..3585bdf 100644 --- a/state/storage.go +++ b/state/storage.go @@ -175,3 +175,7 @@ func (s *Storage) putContractBytecode(contentKey []byte, contentId []byte, conte } return nil } + +func (s *Storage) Close() error { + return s.store.Close() +} diff --git a/storage/content_storage.go b/storage/content_storage.go index ec7cef7..73cb093 100644 --- a/storage/content_storage.go +++ b/storage/content_storage.go @@ -40,6 +40,8 @@ type ContentStorage interface { Put(contentKey []byte, contentId []byte, content []byte) error Radius() *uint256.Int + + Close() error } type MockStorage struct { @@ -67,3 +69,7 @@ func (m *MockStorage) Put(contentKey []byte, contentId []byte, content []byte) e func (m *MockStorage) Radius() *uint256.Int { return uint256.MustFromHex("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") } + +func (m *MockStorage) Close() error { + return nil +} diff --git a/history/storage.go b/storage/sqlite/storage.go similarity index 97% rename from history/storage.go rename to storage/sqlite/storage.go index 47c75a2..f5a7192 100644 --- a/history/storage.go +++ b/storage/sqlite/storage.go @@ -1,4 +1,4 @@ -package history +package sqlite import ( "bytes" @@ -19,7 +19,7 @@ import ( "github.com/holiman/uint256" "github.com/mattn/go-sqlite3" "github.com/optimism-java/shisui2/portalwire" - storage2 "github.com/optimism-java/shisui2/storage" + "github.com/optimism-java/shisui2/storage" ) const ( @@ -43,7 +43,7 @@ const ( ORDER BY distance DESC` ) -var _ storage2.ContentStorage = &ContentStorage{} +var _ storage.ContentStorage = &ContentStorage{} var once sync.Once type ContentStorage struct { @@ -104,14 +104,14 @@ func NewDB(dataDir string, network string) (*sql.DB, error) { return sqlDb, err } -func NewHistoryStorage(config storage2.PortalStorageConfig) (storage2.ContentStorage, error) { +func NewHistoryStorage(config storage.PortalStorageConfig) (storage.ContentStorage, error) { hs := &ContentStorage{ nodeId: config.NodeId, sqliteDB: config.DB, storageCapacityInBytes: config.StorageCapacityMB * 1000000, log: log.New("storage", config.NetworkName), } - hs.radius.Store(storage2.MaxDistance) + hs.radius.Store(storage.MaxDistance) err := hs.createTable() if err != nil { @@ -139,7 +139,7 @@ func (p *ContentStorage) Get(contentKey []byte, contentId []byte) ([]byte, error var res []byte err := p.getStmt.QueryRow(contentId).Scan(&res) if errors.Is(err, sql.ErrNoRows) { - return nil, storage2.ErrContentNotFound + return nil, storage.ErrContentNotFound } return res, err } @@ -370,7 +370,7 @@ func (p *ContentStorage) EstimateNewRadius(currentRadius *uint256.Int) (*uint256 if metrics.Enabled { newRadius := new(uint256.Int).Div(currentRadius, uint256.MustFromBig(bigFormat)) newRadius.Mul(newRadius, uint256.NewInt(100)) - newRadius.Mod(newRadius, storage2.MaxDistance) + newRadius.Mod(newRadius, storage.MaxDistance) portalStorageMetrics.RadiusRatio.Update(newRadius.Float64() / 100) } return new(uint256.Int).Div(currentRadius, uint256.MustFromBig(bigFormat)), nil @@ -466,7 +466,7 @@ func (p *ContentStorage) deleteContentFraction(fraction float64) (deleteCount in p.radius.Store(dis) if metrics.Enabled { dis.Mul(dis, uint256.NewInt(100)) - dis.Mod(dis, storage2.MaxDistance) + dis.Mod(dis, storage.MaxDistance) portalStorageMetrics.RadiusRatio.Update(dis.Float64() / 100) } } diff --git a/history/storage_test.go b/storage/sqlite/storage_test.go similarity index 58% rename from history/storage_test.go rename to storage/sqlite/storage_test.go index 4037ae4..fe0324e 100644 --- a/history/storage_test.go +++ b/storage/sqlite/storage_test.go @@ -1,4 +1,4 @@ -package history +package sqlite import ( "fmt" @@ -8,7 +8,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/enode" "github.com/holiman/uint256" - storage2 "github.com/optimism-java/shisui2/storage" + contentStorage "github.com/optimism-java/shisui2/storage" "github.com/stretchr/testify/assert" ) @@ -26,12 +26,12 @@ func genBytes(length int) []byte { return res } -func newContentStorage(storageCapacityInMB uint64, nodeId enode.ID, nodeDataDir string) (*ContentStorage, error) { - db, err := NewDB(nodeDataDir, "history") +func newContentStorage(storageCapacityInMB uint64, nodeId enode.ID) (*ContentStorage, error) { + db, err := NewDB("./unit_test", "history") if err != nil { return nil, err } - hs, err := NewHistoryStorage(storage2.PortalStorageConfig{ + hs, err := NewHistoryStorage(contentStorage.PortalStorageConfig{ DB: db, StorageCapacityMB: storageCapacityInMB, NodeId: nodeId, @@ -44,67 +44,67 @@ func newContentStorage(storageCapacityInMB uint64, nodeId enode.ID, nodeDataDir func TestBasicStorage(t *testing.T) { zeroNodeId := uint256.NewInt(0).Bytes32() - contentStorage, err := newContentStorage(math.MaxUint32, zeroNodeId, nodeDataDir) + storage, err := newContentStorage(math.MaxUint32, zeroNodeId) assert.NoError(t, err) defer clearNodeData() - defer contentStorage.Close() + defer storage.Close() contentId := []byte("test") content := []byte("value") - _, err = contentStorage.Get(nil, contentId) - assert.Equal(t, storage2.ErrContentNotFound, err) + _, err = storage.Get(nil, contentId) + assert.Equal(t, contentStorage.ErrContentNotFound, err) - pt := contentStorage.put(contentId, content) + pt := storage.put(contentId, content) assert.NoError(t, pt.Err()) - val, err := contentStorage.Get(nil, contentId) + val, err := storage.Get(nil, contentId) assert.NoError(t, err) assert.Equal(t, content, val) - count, err := contentStorage.ContentCount() + count, err := storage.ContentCount() assert.NoError(t, err) assert.Equal(t, count, uint64(1)) - size, err := contentStorage.Size() + size, err := storage.Size() assert.NoError(t, err) assert.True(t, size > 0) - unusedSize, err := contentStorage.UnusedSize() + unusedSize, err := storage.UnusedSize() assert.NoError(t, err) - usedSize, err := contentStorage.UsedSize() + usedSize, err := storage.UsedSize() assert.NoError(t, err) assert.True(t, usedSize == size-unusedSize) } func TestDBSize(t *testing.T) { zeroNodeId := uint256.NewInt(0).Bytes32() - contentStorage, err := newContentStorage(math.MaxUint32, zeroNodeId, nodeDataDir) + storage, err := newContentStorage(math.MaxUint32, zeroNodeId) assert.NoError(t, err) defer clearNodeData() - defer contentStorage.Close() + defer storage.Close() numBytes := 10000 - size1, err := contentStorage.Size() + size1, err := storage.Size() assert.NoError(t, err) - putResult := contentStorage.put(uint256.NewInt(1).Bytes(), genBytes(numBytes)) + putResult := storage.put(uint256.NewInt(1).Bytes(), genBytes(numBytes)) assert.Nil(t, putResult.Err()) - size2, err := contentStorage.Size() + size2, err := storage.Size() assert.NoError(t, err) - putResult = contentStorage.put(uint256.NewInt(2).Bytes(), genBytes(numBytes)) + putResult = storage.put(uint256.NewInt(2).Bytes(), genBytes(numBytes)) assert.NoError(t, putResult.Err()) - size3, err := contentStorage.Size() + size3, err := storage.Size() assert.NoError(t, err) - putResult = contentStorage.put(uint256.NewInt(2).Bytes(), genBytes(numBytes)) + putResult = storage.put(uint256.NewInt(2).Bytes(), genBytes(numBytes)) assert.NoError(t, putResult.Err()) - size4, err := contentStorage.Size() + size4, err := storage.Size() assert.NoError(t, err) - usedSize, err := contentStorage.UsedSize() + usedSize, err := storage.UsedSize() assert.NoError(t, err) assert.True(t, size2 > size1) @@ -112,25 +112,25 @@ func TestDBSize(t *testing.T) { assert.True(t, size4 == size3) assert.True(t, usedSize == size4) - err = contentStorage.del(uint256.NewInt(2).Bytes()) + err = storage.del(uint256.NewInt(2).Bytes()) assert.NoError(t, err) - err = contentStorage.del(uint256.NewInt(1).Bytes()) + err = storage.del(uint256.NewInt(1).Bytes()) assert.NoError(t, err) - usedSize1, err := contentStorage.UsedSize() + usedSize1, err := storage.UsedSize() assert.NoError(t, err) - size5, err := contentStorage.Size() + size5, err := storage.Size() assert.NoError(t, err) assert.True(t, size4 == size5) assert.True(t, usedSize1 < size5) - err = contentStorage.ReclaimSpace() + err = storage.ReclaimSpace() assert.NoError(t, err) - usedSize2, err := contentStorage.UsedSize() + usedSize2, err := storage.UsedSize() assert.NoError(t, err) - size6, err := contentStorage.Size() + size6, err := storage.Size() assert.NoError(t, err) assert.Equal(t, size1, size6) @@ -141,10 +141,10 @@ func TestDBPruning(t *testing.T) { storageCapacity := uint64(1) zeroNodeId := uint256.NewInt(0).Bytes32() - contentStorage, err := newContentStorage(storageCapacity, zeroNodeId, nodeDataDir) + storage, err := newContentStorage(storageCapacity, zeroNodeId) assert.NoError(t, err) defer clearNodeData() - defer contentStorage.Close() + defer storage.Close() furthestElement := uint256.NewInt(40) secondFurthest := uint256.NewInt(30) @@ -152,29 +152,29 @@ func TestDBPruning(t *testing.T) { numBytes := 100_000 // test with private put method - pt1 := contentStorage.put(uint256.NewInt(1).Bytes(), genBytes(numBytes)) + pt1 := storage.put(uint256.NewInt(1).Bytes(), genBytes(numBytes)) assert.NoError(t, pt1.Err()) - pt2 := contentStorage.put(thirdFurthest.Bytes(), genBytes(numBytes)) + pt2 := storage.put(thirdFurthest.Bytes(), genBytes(numBytes)) assert.NoError(t, pt2.Err()) - pt3 := contentStorage.put(uint256.NewInt(3).Bytes(), genBytes(numBytes)) + pt3 := storage.put(uint256.NewInt(3).Bytes(), genBytes(numBytes)) assert.NoError(t, pt3.Err()) - pt4 := contentStorage.put(uint256.NewInt(10).Bytes(), genBytes(numBytes)) + pt4 := storage.put(uint256.NewInt(10).Bytes(), genBytes(numBytes)) assert.NoError(t, pt4.Err()) - pt5 := contentStorage.put(uint256.NewInt(5).Bytes(), genBytes(numBytes)) + pt5 := storage.put(uint256.NewInt(5).Bytes(), genBytes(numBytes)) assert.NoError(t, pt5.Err()) - pt6 := contentStorage.put(uint256.NewInt(11).Bytes(), genBytes(numBytes)) + pt6 := storage.put(uint256.NewInt(11).Bytes(), genBytes(numBytes)) assert.NoError(t, pt6.Err()) - pt7 := contentStorage.put(furthestElement.Bytes(), genBytes(40000)) + pt7 := storage.put(furthestElement.Bytes(), genBytes(40000)) assert.NoError(t, pt7.Err()) - pt8 := contentStorage.put(secondFurthest.Bytes(), genBytes(30000)) + pt8 := storage.put(secondFurthest.Bytes(), genBytes(30000)) assert.NoError(t, pt8.Err()) - pt9 := contentStorage.put(uint256.NewInt(2).Bytes(), genBytes(numBytes*2)) + pt9 := storage.put(uint256.NewInt(2).Bytes(), genBytes(numBytes*2)) assert.NoError(t, pt9.Err()) - res, _ := contentStorage.GetLargestDistance() + res, _ := storage.GetLargestDistance() assert.Equal(t, res, uint256.NewInt(40)) - pt10 := contentStorage.put(uint256.NewInt(4).Bytes(), genBytes(132000)) + pt10 := storage.put(uint256.NewInt(4).Bytes(), genBytes(132000)) assert.NoError(t, pt10.Err()) assert.False(t, pt1.Pruned()) @@ -189,17 +189,17 @@ func TestDBPruning(t *testing.T) { assert.True(t, pt10.Pruned()) assert.Equal(t, pt10.PrunedCount(), 2) - usedSize, err := contentStorage.UsedSize() + usedSize, err := storage.UsedSize() assert.NoError(t, err) - assert.True(t, usedSize < contentStorage.storageCapacityInBytes) + assert.True(t, usedSize < storage.storageCapacityInBytes) - _, err = contentStorage.Get(nil, furthestElement.Bytes()) - assert.Equal(t, storage2.ErrContentNotFound, err) + _, err = storage.Get(nil, furthestElement.Bytes()) + assert.Equal(t, contentStorage.ErrContentNotFound, err) - _, err = contentStorage.Get(nil, secondFurthest.Bytes()) - assert.Equal(t, storage2.ErrContentNotFound, err) + _, err = storage.Get(nil, secondFurthest.Bytes()) + assert.Equal(t, contentStorage.ErrContentNotFound, err) - val, err := contentStorage.Get(nil, thirdFurthest.Bytes()) + val, err := storage.Get(nil, thirdFurthest.Bytes()) assert.NoError(t, err) assert.NotNil(t, val) } @@ -208,23 +208,23 @@ func TestGetLargestDistance(t *testing.T) { storageCapacity := uint64(1) zeroNodeId := uint256.NewInt(0).Bytes32() - contentStorage, err := newContentStorage(storageCapacity, zeroNodeId, nodeDataDir) + storage, err := newContentStorage(storageCapacity, zeroNodeId) assert.NoError(t, err) defer clearNodeData() - defer contentStorage.Close() + defer storage.Close() furthestElement := uint256.NewInt(40) secondFurthest := uint256.NewInt(30) - pt7 := contentStorage.put(furthestElement.Bytes(), genBytes(2000)) + pt7 := storage.put(furthestElement.Bytes(), genBytes(2000)) assert.NoError(t, pt7.Err()) - val, err := contentStorage.Get(nil, furthestElement.Bytes()) + val, err := storage.Get(nil, furthestElement.Bytes()) assert.NoError(t, err) assert.NotNil(t, val) - pt8 := contentStorage.put(secondFurthest.Bytes(), genBytes(2000)) + pt8 := storage.put(secondFurthest.Bytes(), genBytes(2000)) assert.NoError(t, pt8.Err()) - res, err := contentStorage.GetLargestDistance() + res, err := storage.GetLargestDistance() assert.NoError(t, err) assert.Equal(t, furthestElement, res) } @@ -233,37 +233,37 @@ func TestSimpleForcePruning(t *testing.T) { storageCapacity := uint64(100_000) zeroNodeId := uint256.NewInt(0).Bytes32() - contentStorage, err := newContentStorage(storageCapacity, zeroNodeId, nodeDataDir) + storage, err := newContentStorage(storageCapacity, zeroNodeId) assert.NoError(t, err) defer clearNodeData() - defer contentStorage.Close() + defer storage.Close() furthestElement := uint256.NewInt(40) secondFurthest := uint256.NewInt(30) third := uint256.NewInt(10) - pt1 := contentStorage.put(furthestElement.Bytes(), genBytes(2000)) + pt1 := storage.put(furthestElement.Bytes(), genBytes(2000)) assert.NoError(t, pt1.Err()) - pt2 := contentStorage.put(secondFurthest.Bytes(), genBytes(2000)) + pt2 := storage.put(secondFurthest.Bytes(), genBytes(2000)) assert.NoError(t, pt2.Err()) - pt3 := contentStorage.put(third.Bytes(), genBytes(2000)) + pt3 := storage.put(third.Bytes(), genBytes(2000)) assert.NoError(t, pt3.Err()) - res, err := contentStorage.GetLargestDistance() + res, err := storage.GetLargestDistance() assert.NoError(t, err) assert.Equal(t, furthestElement, res) - err = contentStorage.ForcePrune(uint256.NewInt(20)) + err = storage.ForcePrune(uint256.NewInt(20)) assert.NoError(t, err) - _, err = contentStorage.Get(nil, furthestElement.Bytes()) - assert.Equal(t, storage2.ErrContentNotFound, err) + _, err = storage.Get(nil, furthestElement.Bytes()) + assert.Equal(t, contentStorage.ErrContentNotFound, err) - _, err = contentStorage.Get(nil, secondFurthest.Bytes()) - assert.Equal(t, storage2.ErrContentNotFound, err) + _, err = storage.Get(nil, secondFurthest.Bytes()) + assert.Equal(t, contentStorage.ErrContentNotFound, err) - _, err = contentStorage.Get(nil, third.Bytes()) + _, err = storage.Get(nil, third.Bytes()) assert.NoError(t, err) } @@ -277,7 +277,7 @@ func TestForcePruning(t *testing.T) { nodeId := uint256.MustFromHex("0x30994892f3e4889d99deb5340050510d1842778acc7a7948adffa475fed51d6e").Bytes() content := genBytes(1000) - storage, err := newContentStorage(startCap, enode.ID(nodeId), nodeDataDir) + storage, err := newContentStorage(startCap, enode.ID(nodeId)) assert.NoError(t, err) defer clearNodeData() defer storage.Close() diff --git a/storage/tests/storage_bench_test.go b/storage/tests/storage_bench_test.go new file mode 100644 index 0000000..fa6fa3c --- /dev/null +++ b/storage/tests/storage_bench_test.go @@ -0,0 +1,105 @@ +package tests + +import ( + "crypto/rand" + "os" + "testing" + + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/optimism-java/shisui2/storage" + ethpepple "github.com/optimism-java/shisui2/storage/pebble" + "github.com/optimism-java/shisui2/storage/sqlite" +) + +type testCase struct { + name string + dataSize int + numOps int + batchSize int +} + +var testCases = []testCase{ + {"SmallData", 100, 1000, 1}, + {"MediumData", 1024, 1000, 1}, + {"LargeData", 10240, 1000, 1}, + {"SmallBatch", 100, 1000, 100}, + {"MediumBatch", 1024, 1000, 100}, + {"LargeBatch", 10240, 1000, 100}, +} + +func generateTestData(size, count int) ([][]byte, [][]byte, [][]byte) { + keys := make([][]byte, count) + contentIds := make([][]byte, count) + data := make([][]byte, count) + for i := 0; i < count; i++ { + keys[i] = make([]byte, 32) + contentIds[i] = make([]byte, 32) + data[i] = make([]byte, size) + rand.Read(keys[i]) + rand.Read(contentIds[i]) + rand.Read(data[i]) + } + return keys, contentIds, data +} + +func BenchmarkStorageComparison(b *testing.B) { + for _, tc := range testCases { + keys, contentIds, data := generateTestData(tc.dataSize, tc.numOps) + + b.Run("Pebble_"+tc.name, func(b *testing.B) { + dir, _ := os.MkdirTemp("", "pebble-bench-*") + db, _ := ethpepple.NewPeppleDB(dir, 16, 16, "bench") + storage, _ := ethpepple.NewPeppleStorage(ethpepple.PeppleStorageConfig{ + StorageCapacityMB: 1000, + DB: db, + NodeId: enode.ID{}, + NetworkName: "bench", + }) + defer func() { + db.Close() + os.RemoveAll(dir) + }() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + for j := 0; j < tc.numOps; j += tc.batchSize { + for k := 0; k < tc.batchSize && j+k < tc.numOps; k++ { + storage.Put(keys[j+k], contentIds[j+k], data[j+k]) + } + } + + for j := 0; j < tc.numOps; j++ { + storage.Get(keys[j], contentIds[j]) + } + } + }) + + b.Run("SQLite_"+tc.name, func(b *testing.B) { + dir, _ := os.MkdirTemp("", "sqlite-bench-*") + db, _ := sqlite.NewDB(dir, "bench") + storage, _ := sqlite.NewHistoryStorage(storage.PortalStorageConfig{ + StorageCapacityMB: 1000, + DB: db, + NodeId: enode.ID{}, + NetworkName: "bench", + }) + defer func() { + storage.Close() + os.RemoveAll(dir) + }() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + for j := 0; j < tc.numOps; j += tc.batchSize { + for k := 0; k < tc.batchSize && j+k < tc.numOps; k++ { + storage.Put(keys[j+k], contentIds[j+k], data[j+k]) + } + } + + for j := 0; j < tc.numOps; j++ { + storage.Get(keys[j], contentIds[j]) + } + } + }) + } +}