diff --git a/cmd/filenode.go b/cmd/filenode.go index 7f2ebc0..69e5218 100644 --- a/cmd/filenode.go +++ b/cmd/filenode.go @@ -11,8 +11,10 @@ import ( "syscall" "time" + "github.com/anyproto/any-sync/acl" "github.com/anyproto/any-sync/app" "github.com/anyproto/any-sync/app/logger" + "github.com/anyproto/any-sync/consensus/consensusclient" "github.com/anyproto/any-sync/coordinator/coordinatorclient" "github.com/anyproto/any-sync/coordinator/nodeconfsource" "github.com/anyproto/any-sync/metric" @@ -31,7 +33,6 @@ import ( "github.com/anyproto/any-sync-filenode/deletelog" "github.com/anyproto/any-sync-filenode/filenode" "github.com/anyproto/any-sync-filenode/index" - "github.com/anyproto/any-sync-filenode/limit" "github.com/anyproto/any-sync-filenode/redisprovider" // import this to keep govvv in go.mod on mod tidy @@ -112,7 +113,8 @@ func Bootstrap(a *app.App) { Register(secureservice.New()). Register(pool.New()). Register(coordinatorclient.New()). - Register(limit.New()). + Register(consensusclient.New()). + Register(acl.New()). Register(store()). Register(redisprovider.New()). Register(index.New()). diff --git a/config/config.go b/config/config.go index 4728593..42825f5 100644 --- a/config/config.go +++ b/config/config.go @@ -43,56 +43,58 @@ type Config struct { NetworkStorePath string `yaml:"networkStorePath"` NetworkUpdateIntervalSec int `yaml:"networkUpdateIntervalSec"` CafeMigrateKey string `yaml:"cafeMigrateKey"` + DefaultLimit uint64 `yaml:"defaultLimit"` + PersistTtl uint `yaml:"persistTtl"` } func (c *Config) Init(a *app.App) (err error) { return } -func (c Config) Name() (name string) { +func (c *Config) Name() (name string) { return CName } -func (c Config) GetAccount() commonaccount.Config { +func (c *Config) GetAccount() commonaccount.Config { return c.Account } -func (c Config) GetS3Store() s3store.Config { +func (c *Config) GetS3Store() s3store.Config { return c.S3Store } -func (c Config) GetDevStore() FileDevStore { +func (c *Config) GetDevStore() FileDevStore { return c.FileDevStore } -func (c Config) GetDrpc() rpc.Config { +func (c *Config) GetDrpc() rpc.Config { return c.Drpc } -func (c Config) GetMetric() metric.Config { +func (c *Config) GetMetric() metric.Config { return c.Metric } -func (c Config) GetRedis() redisprovider.Config { +func (c *Config) GetRedis() redisprovider.Config { return c.Redis } -func (c Config) GetNodeConf() nodeconf.Configuration { +func (c *Config) GetNodeConf() nodeconf.Configuration { return c.Network } -func (c Config) GetNodeConfStorePath() string { +func (c *Config) GetNodeConfStorePath() string { return c.NetworkStorePath } -func (c Config) GetNodeConfUpdateInterval() int { +func (c *Config) GetNodeConfUpdateInterval() int { return c.NetworkUpdateIntervalSec } -func (c Config) GetYamux() yamux.Config { +func (c *Config) GetYamux() yamux.Config { return c.Yamux } -func (c Config) GetQuic() quic.Config { +func (c *Config) GetQuic() quic.Config { return c.Quic } diff --git a/etc/any-sync-filenode.yml b/etc/any-sync-filenode.yml index babba9c..3b08ad0 100755 --- a/etc/any-sync-filenode.yml +++ b/etc/any-sync-filenode.yml @@ -63,3 +63,4 @@ network: creationTime: 2023-04-13T20:29:23.453806629+02:00 networkStorePath: . networkUpdateIntervalSec: 600 +defaultLimit: 1073741824 diff --git a/filenode/filenode.go b/filenode/filenode.go index 17a2af9..cc56410 100644 --- a/filenode/filenode.go +++ b/filenode/filenode.go @@ -3,21 +3,24 @@ package filenode import ( "context" "errors" + "slices" + "github.com/anyproto/any-sync/acl" "github.com/anyproto/any-sync/app" "github.com/anyproto/any-sync/app/logger" "github.com/anyproto/any-sync/commonfile/fileblockstore" "github.com/anyproto/any-sync/commonfile/fileproto" "github.com/anyproto/any-sync/commonfile/fileproto/fileprotoerr" "github.com/anyproto/any-sync/metric" + "github.com/anyproto/any-sync/net/peer" "github.com/anyproto/any-sync/net/rpc/server" + "github.com/anyproto/any-sync/nodeconf" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "go.uber.org/zap" "github.com/anyproto/any-sync-filenode/config" "github.com/anyproto/any-sync-filenode/index" - "github.com/anyproto/any-sync-filenode/limit" "github.com/anyproto/any-sync-filenode/store" ) @@ -38,21 +41,23 @@ type Service interface { } type fileNode struct { + acl acl.AclService index index.Index store store.Store - limit limit.Limit metric metric.Metric + nodeConf nodeconf.Service migrateKey string handler *rpcHandler } func (fn *fileNode) Init(a *app.App) (err error) { + fn.acl = a.MustComponent(acl.CName).(acl.AclService) fn.store = a.MustComponent(fileblockstore.CName).(store.Store) fn.index = a.MustComponent(index.CName).(index.Index) - fn.limit = a.MustComponent(limit.CName).(limit.Limit) fn.handler = &rpcHandler{f: fn} fn.metric = a.MustComponent(metric.CName).(metric.Metric) fn.migrateKey = a.MustComponent(config.CName).(*config.Config).CafeMigrateKey + fn.nodeConf = a.MustComponent(nodeconf.CName).(nodeconf.Service) return fileproto.DRPCRegisterFile(a.MustComponent(server.CName).(server.DRPCServer), fn.handler) } @@ -60,13 +65,19 @@ func (fn *fileNode) Name() (name string) { return CName } -func (fn *fileNode) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) { - exists, err := fn.index.CidExists(ctx, k) - if err != nil { - return nil, err - } - if !exists { - return nil, fileprotoerr.ErrCIDNotFound +func (fn *fileNode) Get(ctx context.Context, k cid.Cid, wait bool) (blocks.Block, error) { + if wait { + if err := fn.index.WaitCidExists(ctx, k); err != nil { + return nil, err + } + } else { + exists, err := fn.index.CidExists(ctx, k) + if err != nil { + return nil, err + } + if !exists { + return nil, fileprotoerr.ErrCIDNotFound + } } return fn.store.Get(ctx, k) } @@ -95,6 +106,7 @@ func (fn *fileNode) Add(ctx context.Context, spaceId string, fileId string, bs [ if err = fn.index.BlocksAdd(ctx, bs); err != nil { return err } + fn.index.OnBlockUploaded(ctx, bs...) } cidEntries, err := fn.index.CidEntriesByBlocks(ctx, bs) if err != nil { @@ -159,46 +171,56 @@ func (fn *fileNode) StoreKey(ctx context.Context, spaceId string, checkLimit boo if spaceId == "" { return storageKey, fileprotoerr.ErrForbidden } - // this call also confirms that space exists and valid - limitBytes, groupId, err := fn.limit.Check(ctx, spaceId) + + identity, err := peer.CtxPubKey(ctx) if err != nil { - return + return storageKey, fileprotoerr.ErrForbidden } + ownerPubKey, err := fn.acl.OwnerPubKey(ctx, spaceId) + if err != nil { + log.WarnCtx(ctx, "acl ownerPubKey error", zap.Error(err)) + return storageKey, fileprotoerr.ErrForbidden + } storageKey = index.Key{ - GroupId: groupId, + GroupId: ownerPubKey.Account(), SpaceId: spaceId, } + // if it not owner + if identity.Account() != storageKey.GroupId { + permissions, err := fn.acl.Permissions(ctx, identity, spaceId) + if err != nil { + log.WarnCtx(ctx, "acl permissions error", zap.Error(err)) + return storageKey, fileprotoerr.ErrForbidden + } + if !permissions.CanWrite() { + return storageKey, fileprotoerr.ErrForbidden + } + } + if e := fn.index.Migrate(ctx, storageKey); e != nil { log.WarnCtx(ctx, "space migrate error", zap.String("spaceId", spaceId), zap.Error(e)) } if checkLimit { - info, e := fn.index.GroupInfo(ctx, groupId) - if e != nil { - return storageKey, e - } - if info.BytesUsage >= limitBytes { - return storageKey, fileprotoerr.ErrSpaceLimitExceeded + if err = fn.index.CheckLimits(ctx, storageKey); err != nil { + if errors.Is(err, index.ErrLimitExceed) { + return storageKey, fileprotoerr.ErrSpaceLimitExceeded + } else { + log.WarnCtx(ctx, "check limit error", zap.Error(err)) + return storageKey, fileprotoerr.ErrUnexpected + } } } return } func (fn *fileNode) SpaceInfo(ctx context.Context, spaceId string) (info *fileproto.SpaceInfoResponse, err error) { - var ( - storageKey = index.Key{SpaceId: spaceId} - limitBytes uint64 - ) - if limitBytes, storageKey.GroupId, err = fn.limit.Check(ctx, spaceId); err != nil { + storageKey, err := fn.StoreKey(ctx, spaceId, false) + if err != nil { return nil, err } - - if e := fn.index.Migrate(ctx, storageKey); e != nil { - log.WarnCtx(ctx, "space migrate error", zap.String("spaceId", spaceId), zap.Error(e)) - } - groupInfo, err := fn.index.GroupInfo(ctx, storageKey.GroupId) if err != nil { return nil, err @@ -206,31 +228,32 @@ func (fn *fileNode) SpaceInfo(ctx context.Context, spaceId string) (info *filepr if info, err = fn.spaceInfo(ctx, storageKey, groupInfo); err != nil { return nil, err } - info.LimitBytes = limitBytes return } func (fn *fileNode) AccountInfo(ctx context.Context) (info *fileproto.AccountInfoResponse, err error) { info = &fileproto.AccountInfoResponse{} - // we have space/identity validation in limit.Check - var groupId string - if info.LimitBytes, groupId, err = fn.limit.Check(ctx, ""); err != nil { - return nil, err + identity, err := peer.CtxPubKey(ctx) + if err != nil { + return nil, fileprotoerr.ErrForbidden } + groupId := identity.Account() + groupInfo, err := fn.index.GroupInfo(ctx, groupId) if err != nil { return nil, err } info.TotalCidsCount = groupInfo.CidsCount info.TotalUsageBytes = groupInfo.BytesUsage + info.LimitBytes = groupInfo.Limit + info.AccountLimitBytes = groupInfo.AccountLimit for _, spaceId := range groupInfo.SpaceIds { spaceInfo, err := fn.spaceInfo(ctx, index.Key{GroupId: groupId, SpaceId: spaceId}, groupInfo) if err != nil { return nil, err } - spaceInfo.LimitBytes = info.LimitBytes info.Spaces = append(info.Spaces, spaceInfo) } return @@ -243,7 +266,13 @@ func (fn *fileNode) spaceInfo(ctx context.Context, key index.Key, groupInfo inde if err != nil { return nil, err } - info.TotalUsageBytes = groupInfo.BytesUsage + if spaceInfo.Limit == 0 { + info.TotalUsageBytes = groupInfo.BytesUsage + info.LimitBytes = groupInfo.Limit + } else { + info.TotalUsageBytes = spaceInfo.BytesUsage + info.LimitBytes = spaceInfo.Limit + } info.FilesCount = uint64(spaceInfo.FileCount) info.CidsCount = spaceInfo.CidsCount info.SpaceUsageBytes = spaceInfo.BytesUsage @@ -293,3 +322,33 @@ func (fn *fileNode) MigrateCafe(ctx context.Context, bs []blocks.Block) error { } return nil } + +func (fn *fileNode) AccountLimitSet(ctx context.Context, identity string, limit uint64) (err error) { + peerId, err := peer.CtxPeerId(ctx) + if err != nil { + return + } + // check that call from the coordinator or the payment node + if !slices.Contains(fn.nodeConf.NodeTypes(peerId), nodeconf.NodeTypeCoordinator) && + !slices.Contains(fn.nodeConf.NodeTypes(peerId), nodeconf.NodeTypePaymentProcessingNode) { + return fileprotoerr.ErrForbidden + } + + return fn.index.SetGroupLimit(ctx, identity, limit) +} + +func (fn *fileNode) SpaceLimitSet(ctx context.Context, spaceId string, limit uint64) (err error) { + storeKey, err := fn.StoreKey(ctx, spaceId, false) + if err != nil { + return + } + return fn.index.SetSpaceLimit(ctx, storeKey, limit) +} + +func (fn *fileNode) FilesGet(ctx context.Context, spaceId string) (fileIds []string, err error) { + storeKey, err := fn.StoreKey(ctx, spaceId, false) + if err != nil { + return + } + return fn.index.FilesList(ctx, storeKey) +} diff --git a/filenode/filenode_test.go b/filenode/filenode_test.go index e03e34c..7cec677 100644 --- a/filenode/filenode_test.go +++ b/filenode/filenode_test.go @@ -4,12 +4,18 @@ import ( "context" "testing" + "github.com/anyproto/any-sync/acl" + "github.com/anyproto/any-sync/acl/mock_acl" "github.com/anyproto/any-sync/app" "github.com/anyproto/any-sync/commonfile/fileblockstore" "github.com/anyproto/any-sync/commonfile/fileproto" "github.com/anyproto/any-sync/commonfile/fileproto/fileprotoerr" "github.com/anyproto/any-sync/metric" + "github.com/anyproto/any-sync/net/peer" "github.com/anyproto/any-sync/net/rpc/server" + "github.com/anyproto/any-sync/nodeconf" + "github.com/anyproto/any-sync/nodeconf/mock_nodeconf" + "github.com/anyproto/any-sync/util/crypto" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/stretchr/testify/assert" @@ -19,33 +25,30 @@ import ( "github.com/anyproto/any-sync-filenode/config" "github.com/anyproto/any-sync-filenode/index" "github.com/anyproto/any-sync-filenode/index/mock_index" - "github.com/anyproto/any-sync-filenode/limit" - "github.com/anyproto/any-sync-filenode/limit/mock_limit" "github.com/anyproto/any-sync-filenode/store/mock_store" "github.com/anyproto/any-sync-filenode/testutil" ) -var ctx = context.Background() - func TestFileNode_Add(t *testing.T) { t.Run("success", func(t *testing.T) { fx := newFixture(t) defer fx.Finish(t) var ( - storeKey = newRandKey() - fileId = testutil.NewRandCid().String() - b = testutil.NewRandBlock(1024) + ctx, storeKey = newRandKey() + fileId = testutil.NewRandCid().String() + b = testutil.NewRandBlock(1024) ) - fx.limit.EXPECT().Check(ctx, storeKey.SpaceId).Return(uint64(123), storeKey.GroupId, nil) + fx.aclService.EXPECT().OwnerPubKey(ctx, storeKey.SpaceId).Return(mustPubKey(ctx), nil) + fx.index.EXPECT().CheckLimits(ctx, storeKey) fx.index.EXPECT().Migrate(ctx, storeKey) - fx.index.EXPECT().GroupInfo(ctx, storeKey.GroupId).Return(index.GroupInfo{BytesUsage: uint64(120)}, nil) fx.index.EXPECT().BlocksLock(ctx, []blocks.Block{b}).Return(func() {}, nil) fx.index.EXPECT().BlocksGetNonExistent(ctx, []blocks.Block{b}).Return([]blocks.Block{b}, nil) fx.store.EXPECT().Add(ctx, []blocks.Block{b}) fx.index.EXPECT().BlocksAdd(ctx, []blocks.Block{b}) fx.index.EXPECT().CidEntriesByBlocks(ctx, []blocks.Block{b}).Return(&index.CidEntries{}, nil) fx.index.EXPECT().FileBind(ctx, storeKey, fileId, gomock.Any()) + fx.index.EXPECT().OnBlockUploaded(ctx, []blocks.Block{b}) resp, err := fx.handler.BlockPush(ctx, &fileproto.BlockPushRequest{ SpaceId: storeKey.SpaceId, @@ -61,14 +64,14 @@ func TestFileNode_Add(t *testing.T) { fx := newFixture(t) defer fx.Finish(t) var ( - storeKey = newRandKey() - fileId = testutil.NewRandCid().String() - b = testutil.NewRandBlock(1024) + ctx, storeKey = newRandKey() + fileId = testutil.NewRandCid().String() + b = testutil.NewRandBlock(1024) ) - fx.limit.EXPECT().Check(ctx, storeKey.SpaceId).Return(uint64(123), storeKey.GroupId, nil) + fx.aclService.EXPECT().OwnerPubKey(ctx, storeKey.SpaceId).Return(mustPubKey(ctx), nil) fx.index.EXPECT().Migrate(ctx, storeKey) - fx.index.EXPECT().GroupInfo(ctx, storeKey.GroupId).Return(index.GroupInfo{BytesUsage: uint64(124)}, nil) + fx.index.EXPECT().CheckLimits(ctx, storeKey).Return(index.ErrLimitExceed) resp, err := fx.handler.BlockPush(ctx, &fileproto.BlockPushRequest{ SpaceId: storeKey.SpaceId, @@ -83,10 +86,10 @@ func TestFileNode_Add(t *testing.T) { fx := newFixture(t) defer fx.Finish(t) var ( - storeKey = newRandKey() - fileId = testutil.NewRandCid().String() - b = testutil.NewRandBlock(1024) - b2 = testutil.NewRandBlock(10) + ctx, storeKey = newRandKey() + fileId = testutil.NewRandCid().String() + b = testutil.NewRandBlock(1024) + b2 = testutil.NewRandBlock(10) ) resp, err := fx.handler.BlockPush(ctx, &fileproto.BlockPushRequest{ @@ -102,9 +105,10 @@ func TestFileNode_Add(t *testing.T) { fx := newFixture(t) defer fx.Finish(t) var ( - spaceId = testutil.NewRandSpaceId() - fileId = testutil.NewRandCid().String() - b = testutil.NewRandBlock(cidSizeLimit + 1) + ctx, key = newRandKey() + spaceId = key.SpaceId + fileId = testutil.NewRandCid().String() + b = testutil.NewRandBlock(cidSizeLimit + 1) ) resp, err := fx.handler.BlockPush(ctx, &fileproto.BlockPushRequest{ @@ -123,7 +127,8 @@ func TestFileNode_Get(t *testing.T) { t.Run("success", func(t *testing.T) { fx := newFixture(t) defer fx.Finish(t) - spaceId := testutil.NewRandSpaceId() + ctx, key := newRandKey() + spaceId := key.SpaceId b := testutil.NewRandBlock(10) fx.index.EXPECT().CidExists(gomock.Any(), b.Cid()).Return(true, nil) fx.store.EXPECT().Get(ctx, b.Cid()).Return(b, nil) @@ -137,7 +142,8 @@ func TestFileNode_Get(t *testing.T) { t.Run("not found", func(t *testing.T) { fx := newFixture(t) defer fx.Finish(t) - spaceId := testutil.NewRandSpaceId() + ctx, key := newRandKey() + spaceId := key.SpaceId b := testutil.NewRandBlock(10) fx.index.EXPECT().CidExists(gomock.Any(), b.Cid()).Return(false, nil) resp, err := fx.handler.BlockGet(ctx, &fileproto.BlockGetRequest{ @@ -152,13 +158,13 @@ func TestFileNode_Get(t *testing.T) { func TestFileNode_Check(t *testing.T) { fx := newFixture(t) defer fx.Finish(t) - var storeKey = newRandKey() + var ctx, storeKey = newRandKey() var bs = testutil.NewRandBlocks(3) cids := make([][]byte, len(bs)) for _, b := range bs { cids = append(cids, b.Cid().Bytes()) } - fx.limit.EXPECT().Check(ctx, storeKey.SpaceId).Return(uint64(100000), storeKey.GroupId, nil) + fx.aclService.EXPECT().OwnerPubKey(ctx, storeKey.SpaceId).Return(mustPubKey(ctx), nil) fx.index.EXPECT().Migrate(ctx, storeKey) fx.index.EXPECT().CidExistsInSpace(ctx, storeKey, testutil.BlocksToKeys(bs)).Return(testutil.BlocksToKeys(bs[:1]), nil) fx.index.EXPECT().CidExists(ctx, bs[1].Cid()).Return(true, nil) @@ -178,21 +184,21 @@ func TestFileNode_BlocksBind(t *testing.T) { fx := newFixture(t) defer fx.Finish(t) var ( - storeKey = newRandKey() - fileId = testutil.NewRandCid().String() - bs = testutil.NewRandBlocks(3) - cidsB = make([][]byte, len(bs)) - cids = make([]cid.Cid, len(bs)) - cidEntries = &index.CidEntries{} + ctx, storeKey = newRandKey() + fileId = testutil.NewRandCid().String() + bs = testutil.NewRandBlocks(3) + cidsB = make([][]byte, len(bs)) + cids = make([]cid.Cid, len(bs)) + cidEntries = &index.CidEntries{} ) for i, b := range bs { cids[i] = b.Cid() cidsB[i] = b.Cid().Bytes() } - fx.limit.EXPECT().Check(ctx, storeKey.SpaceId).Return(uint64(123), storeKey.GroupId, nil) + fx.aclService.EXPECT().OwnerPubKey(ctx, storeKey.SpaceId).Return(mustPubKey(ctx), nil) + fx.index.EXPECT().CheckLimits(ctx, storeKey) fx.index.EXPECT().Migrate(ctx, storeKey) - fx.index.EXPECT().GroupInfo(ctx, storeKey.GroupId).Return(index.GroupInfo{BytesUsage: 12}, nil) fx.index.EXPECT().CidEntries(ctx, cids).Return(cidEntries, nil) fx.index.EXPECT().FileBind(ctx, storeKey, fileId, cidEntries) @@ -211,11 +217,11 @@ func TestFileNode_FileInfo(t *testing.T) { defer fx.Finish(t) var ( - storeKey = newRandKey() - fileId1 = testutil.NewRandCid().String() - fileId2 = testutil.NewRandCid().String() + ctx, storeKey = newRandKey() + fileId1 = testutil.NewRandCid().String() + fileId2 = testutil.NewRandCid().String() ) - fx.limit.EXPECT().Check(ctx, storeKey.SpaceId).AnyTimes().Return(uint64(100000), storeKey.GroupId, nil) + fx.aclService.EXPECT().OwnerPubKey(ctx, storeKey.SpaceId).Return(mustPubKey(ctx), nil) fx.index.EXPECT().Migrate(ctx, storeKey) fx.index.EXPECT().FileInfo(ctx, storeKey, fileId1, fileId2).Return([]index.FileInfo{{1, 1}, {2, 2}}, nil) @@ -234,44 +240,123 @@ func TestFileNode_AccountInfo(t *testing.T) { defer fx.Finish(t) var ( - storeKey = newRandKey() + ctx, storeKey = newRandKey() + secondSpaceId = testutil.NewRandSpaceId() ) - fx.limit.EXPECT().Check(ctx, "").AnyTimes().Return(uint64(100000), storeKey.GroupId, nil) fx.index.EXPECT().GroupInfo(ctx, storeKey.GroupId).Return(index.GroupInfo{ - BytesUsage: 100, - CidsCount: 10, - SpaceIds: []string{storeKey.SpaceId}, + BytesUsage: 100, + CidsCount: 10, + SpaceIds: []string{storeKey.SpaceId, secondSpaceId}, + Limit: 90000, + AccountLimit: 100000, }, nil) fx.index.EXPECT().SpaceInfo(ctx, storeKey).Return(index.SpaceInfo{ BytesUsage: 90, CidsCount: 9, FileCount: 1, }, nil) + fx.index.EXPECT().SpaceInfo(ctx, index.Key{GroupId: storeKey.GroupId, SpaceId: secondSpaceId}).Return(index.SpaceInfo{ + BytesUsage: 80, + CidsCount: 8, + FileCount: 2, + Limit: 100, + }, nil) resp, err := fx.handler.AccountInfo(ctx, &fileproto.AccountInfoRequest{}) require.NoError(t, err) - require.Len(t, resp.Spaces, 1) + require.Len(t, resp.Spaces, 2) assert.Equal(t, uint64(100), resp.TotalUsageBytes) assert.Equal(t, uint64(10), resp.TotalCidsCount) - assert.Equal(t, uint64(100000), resp.LimitBytes) + assert.Equal(t, uint64(90000), resp.LimitBytes) + assert.Equal(t, uint64(100000), resp.AccountLimitBytes) assert.Equal(t, uint64(90), resp.Spaces[0].SpaceUsageBytes) assert.Equal(t, uint64(9), resp.Spaces[0].CidsCount) assert.Equal(t, uint64(1), resp.Spaces[0].FilesCount) - assert.Equal(t, uint64(100000), resp.Spaces[0].LimitBytes) + assert.Equal(t, uint64(90000), resp.Spaces[0].LimitBytes) assert.Equal(t, uint64(100), resp.Spaces[0].TotalUsageBytes) + + assert.Equal(t, uint64(80), resp.Spaces[1].SpaceUsageBytes) + assert.Equal(t, uint64(8), resp.Spaces[1].CidsCount) + assert.Equal(t, uint64(2), resp.Spaces[1].FilesCount) + assert.Equal(t, uint64(100), resp.Spaces[1].LimitBytes) + assert.Equal(t, uint64(80), resp.Spaces[1].TotalUsageBytes) +} + +func TestFileNode_SpaceInfo(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + + var ( + ctx, storeKey = newRandKey() + ) + fx.aclService.EXPECT().OwnerPubKey(ctx, storeKey.SpaceId).Return(mustPubKey(ctx), nil) + fx.index.EXPECT().Migrate(ctx, storeKey) + + fx.index.EXPECT().GroupInfo(ctx, storeKey.GroupId).Return(index.GroupInfo{ + BytesUsage: 100, + CidsCount: 10, + SpaceIds: []string{storeKey.SpaceId}, + Limit: 90000, + AccountLimit: 100000, + }, nil) + fx.index.EXPECT().SpaceInfo(ctx, storeKey).Return(index.SpaceInfo{ + BytesUsage: 90, + CidsCount: 9, + FileCount: 1, + }, nil) + + info, err := fx.SpaceInfo(ctx, storeKey.SpaceId) + require.NoError(t, err) + require.NotNil(t, info) + assert.Equal(t, uint64(90), info.SpaceUsageBytes) + assert.Equal(t, uint64(9), info.CidsCount) + assert.Equal(t, uint64(1), info.FilesCount) + assert.Equal(t, uint64(90000), info.LimitBytes) + assert.Equal(t, uint64(100), info.TotalUsageBytes) +} + +func TestFileNode_AccountLimitSet(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + + var peerId = "peerId" + + fx.nodeConf.EXPECT().NodeTypes(peerId).Return([]nodeconf.NodeType{nodeconf.NodeTypeCoordinator}) + + ctx := peer.CtxWithPeerId(context.Background(), peerId) + + _, storeKey := newRandKey() + identity := storeKey.GroupId + + fx.index.EXPECT().SetGroupLimit(ctx, identity, uint64(12345)) + + require.NoError(t, fx.AccountLimitSet(ctx, identity, 12345)) +} + +func TestFileNode_SpaceLimitSet(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + + ctx, storeKey := newRandKey() + + fx.aclService.EXPECT().OwnerPubKey(ctx, storeKey.SpaceId).Return(mustPubKey(ctx), nil) + fx.index.EXPECT().Migrate(ctx, storeKey) + fx.index.EXPECT().SetSpaceLimit(ctx, storeKey, uint64(12345)) + require.NoError(t, fx.SpaceLimitSet(ctx, storeKey.SpaceId, 12345)) } func newFixture(t *testing.T) *fixture { ctrl := gomock.NewController(t) fx := &fixture{ - fileNode: New().(*fileNode), - index: mock_index.NewMockIndex(ctrl), - store: mock_store.NewMockStore(ctrl), - limit: mock_limit.NewMockLimit(ctrl), - serv: server.New(), - ctrl: ctrl, - a: new(app.App), + fileNode: New().(*fileNode), + index: mock_index.NewMockIndex(ctrl), + store: mock_store.NewMockStore(ctrl), + aclService: mock_acl.NewMockAclService(ctrl), + nodeConf: mock_nodeconf.NewMockService(ctrl), + serv: server.New(), + ctrl: ctrl, + a: new(app.App), } fx.index.EXPECT().Name().Return(index.CName).AnyTimes() @@ -282,34 +367,57 @@ func newFixture(t *testing.T) *fixture { fx.store.EXPECT().Name().Return(fileblockstore.CName).AnyTimes() fx.store.EXPECT().Init(gomock.Any()).AnyTimes() - fx.limit.EXPECT().Name().Return(limit.CName).AnyTimes() - fx.limit.EXPECT().Init(gomock.Any()).AnyTimes() - fx.limit.EXPECT().Run(gomock.Any()).AnyTimes() - fx.limit.EXPECT().Close(gomock.Any()).AnyTimes() - - fx.a.Register(metric.New()).Register(fx.serv).Register(fx.index).Register(fx.store).Register(fx.limit).Register(fx.fileNode).Register(&config.Config{}) - require.NoError(t, fx.a.Start(ctx)) + fx.aclService.EXPECT().Name().Return(acl.CName).AnyTimes() + fx.aclService.EXPECT().Init(gomock.Any()).AnyTimes() + fx.aclService.EXPECT().Run(gomock.Any()).AnyTimes() + fx.aclService.EXPECT().Close(gomock.Any()).AnyTimes() + + fx.nodeConf.EXPECT().Name().Return(nodeconf.CName).AnyTimes() + fx.nodeConf.EXPECT().Init(gomock.Any()).AnyTimes() + fx.nodeConf.EXPECT().Run(gomock.Any()).AnyTimes() + fx.nodeConf.EXPECT().Close(gomock.Any()).AnyTimes() + + fx.a.Register(metric.New()). + Register(fx.serv). + Register(fx.index). + Register(fx.store). + Register(fx.aclService). + Register(fx.fileNode). + Register(fx.nodeConf). + Register(&config.Config{}) + require.NoError(t, fx.a.Start(context.Background())) return fx } type fixture struct { *fileNode - index *mock_index.MockIndex - store *mock_store.MockStore - ctrl *gomock.Controller - a *app.App - limit *mock_limit.MockLimit - serv server.DRPCServer + index *mock_index.MockIndex + store *mock_store.MockStore + ctrl *gomock.Controller + a *app.App + aclService *mock_acl.MockAclService + serv server.DRPCServer + nodeConf *mock_nodeconf.MockService } func (fx *fixture) Finish(t *testing.T) { fx.ctrl.Finish() - require.NoError(t, fx.a.Close(ctx)) + require.NoError(t, fx.a.Close(context.Background())) } -func newRandKey() index.Key { - return index.Key{ +func newRandKey() (context.Context, index.Key) { + _, pubKey, _ := crypto.GenerateRandomEd25519KeyPair() + pubKeyRaw, _ := pubKey.Marshall() + return peer.CtxWithIdentity(context.Background(), pubKeyRaw), index.Key{ SpaceId: testutil.NewRandSpaceId(), - GroupId: "A" + testutil.NewRandCid().String(), + GroupId: pubKey.Account(), + } +} + +func mustPubKey(ctx context.Context) crypto.PubKey { + pubKey, err := peer.CtxPubKey(ctx) + if err != nil { + panic(err) } + return pubKey } diff --git a/filenode/rpchandler.go b/filenode/rpchandler.go index 761faad..5746350 100644 --- a/filenode/rpchandler.go +++ b/filenode/rpchandler.go @@ -35,6 +35,7 @@ func (r rpcHandler) BlockGet(ctx context.Context, req *fileproto.BlockGetRequest metric.SpaceId(req.SpaceId), metric.Size(size), metric.Cid(c.String()), + zap.Bool("wait", req.Wait), zap.Error(err), ) }() @@ -45,7 +46,7 @@ func (r rpcHandler) BlockGet(ctx context.Context, req *fileproto.BlockGetRequest if err != nil { return nil, err } - b, err := r.f.Get(ctx, c) + b, err := r.f.Get(ctx, c, req.Wait) if err != nil { return nil, err } else { @@ -54,7 +55,7 @@ func (r rpcHandler) BlockGet(ctx context.Context, req *fileproto.BlockGetRequest return resp, nil } -func (r rpcHandler) BlockPush(ctx context.Context, req *fileproto.BlockPushRequest) (resp *fileproto.BlockPushResponse, err error) { +func (r rpcHandler) BlockPush(ctx context.Context, req *fileproto.BlockPushRequest) (resp *fileproto.Ok, err error) { var c cid.Cid st := time.Now() defer func() { @@ -91,7 +92,7 @@ func (r rpcHandler) BlockPush(ctx context.Context, req *fileproto.BlockPushReque if err = r.f.Add(ctx, req.SpaceId, req.FileId, []blocks.Block{b}); err != nil { return nil, err } - return &fileproto.BlockPushResponse{}, nil + return &fileproto.Ok{}, nil } func (r rpcHandler) BlocksCheck(ctx context.Context, req *fileproto.BlocksCheckRequest) (resp *fileproto.BlocksCheckResponse, err error) { @@ -114,7 +115,7 @@ func (r rpcHandler) BlocksCheck(ctx context.Context, req *fileproto.BlocksCheckR }, nil } -func (r rpcHandler) BlocksBind(ctx context.Context, req *fileproto.BlocksBindRequest) (resp *fileproto.BlocksBindResponse, err error) { +func (r rpcHandler) BlocksBind(ctx context.Context, req *fileproto.BlocksBindRequest) (resp *fileproto.Ok, err error) { st := time.Now() defer func() { r.f.metric.RequestLog(ctx, @@ -129,7 +130,7 @@ func (r rpcHandler) BlocksBind(ctx context.Context, req *fileproto.BlocksBindReq if err = r.f.BlocksBind(ctx, req.SpaceId, req.FileId, convertCids(req.Cids)...); err != nil { return nil, err } - return &fileproto.BlocksBindResponse{}, nil + return &fileproto.Ok{}, nil } func (r rpcHandler) FilesDelete(ctx context.Context, req *fileproto.FilesDeleteRequest) (resp *fileproto.FilesDeleteResponse, err error) { @@ -172,6 +173,33 @@ func (r rpcHandler) FilesInfo(ctx context.Context, req *fileproto.FilesInfoReque return resp, nil } +func (r rpcHandler) FilesGet(req *fileproto.FilesGetRequest, stream fileproto.DRPCFile_FilesGetStream) (err error) { + ctx := stream.Context() + st := time.Now() + defer func() { + r.f.metric.RequestLog(ctx, + "file.filesGet", + metric.TotalDur(time.Since(st)), + metric.SpaceId(req.SpaceId), + zap.Error(err), + ) + }() + + fileIds, err := r.f.FilesGet(ctx, req.SpaceId) + if err != nil { + return + } + + for _, fileId := range fileIds { + if err = stream.Send(&fileproto.FilesGetResponse{ + FileId: fileId, + }); err != nil { + return + } + } + return +} + func (r rpcHandler) Check(ctx context.Context, req *fileproto.CheckRequest) (*fileproto.CheckResponse, error) { st := time.Now() defer func() { @@ -217,6 +245,36 @@ func (r rpcHandler) AccountInfo(ctx context.Context, req *fileproto.AccountInfoR return } +func (r rpcHandler) AccountLimitSet(ctx context.Context, req *fileproto.AccountLimitSetRequest) (resp *fileproto.Ok, err error) { + st := time.Now() + defer func() { + r.f.metric.RequestLog(ctx, + "file.accountLimitSet", + metric.TotalDur(time.Since(st)), + zap.Error(err), + ) + }() + if err = r.f.AccountLimitSet(ctx, req.Identity, req.Limit); err != nil { + return + } + return &fileproto.Ok{}, nil +} + +func (r rpcHandler) SpaceLimitSet(ctx context.Context, req *fileproto.SpaceLimitSetRequest) (resp *fileproto.Ok, err error) { + st := time.Now() + defer func() { + r.f.metric.RequestLog(ctx, + "file.spaceLimitSet", + metric.TotalDur(time.Since(st)), + zap.Error(err), + ) + }() + if err = r.f.SpaceLimitSet(ctx, req.SpaceId, req.Limit); err != nil { + return + } + return &fileproto.Ok{}, nil +} + func convertCids(bCids [][]byte) (cids []cid.Cid) { cids = make([]cid.Cid, 0, len(bCids)) var uniqMap map[string]struct{} diff --git a/go.mod b/go.mod index cd51ff7..4f743c1 100644 --- a/go.mod +++ b/go.mod @@ -5,50 +5,61 @@ go 1.21 require ( github.com/OneOfOne/xxhash v1.2.8 github.com/ahmetb/govvv v0.3.0 - github.com/anyproto/any-sync v0.3.7 + github.com/anyproto/any-sync v0.3.33 github.com/aws/aws-sdk-go v1.51.1 github.com/cespare/xxhash/v2 v2.2.0 github.com/go-redsync/redsync/v4 v4.12.1 github.com/gogo/protobuf v1.3.2 - github.com/golang/snappy v0.0.4 + github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/redis/go-redis/v9 v9.5.1 github.com/stretchr/testify v1.9.0 - go.uber.org/atomic v1.11.0 go.uber.org/mock v0.4.0 go.uber.org/zap v1.27.0 gopkg.in/yaml.v3 v3.0.1 ) require ( - filippo.io/edwards25519 v1.0.0 // indirect + filippo.io/edwards25519 v1.1.0 // indirect github.com/anyproto/go-chash v0.1.0 // indirect github.com/anyproto/go-slip10 v1.0.0 // indirect github.com/anyproto/go-slip21 v1.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bits-and-blooms/bitset v1.10.0 // indirect + github.com/btcsuite/btcd v0.22.1 // indirect + github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect + github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect + github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce // indirect github.com/cespare/xxhash v1.1.0 // indirect + github.com/cheggaaa/mb/v3 v3.0.2 // indirect + github.com/consensys/bavard v0.1.13 // indirect + github.com/consensys/gnark-crypto v0.12.1 // indirect + github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/ethereum/c-kzg-4844 v0.4.0 // indirect + github.com/ethereum/go-ethereum v1.13.12 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gobwas/glob v0.2.3 // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b // indirect + github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/yamux v0.1.1 // indirect + github.com/holiman/uint256 v1.2.4 // indirect github.com/ipfs/go-ipfs-util v0.0.2 // indirect github.com/ipfs/go-log/v2 v2.5.1 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect - github.com/libp2p/go-libp2p v0.32.0 // indirect + github.com/libp2p/go-libp2p v0.32.2 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/miguelmota/go-ethereum-hdwallet v0.1.2 // indirect github.com/minio/sha256-simd v1.0.1 // indirect + github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect @@ -58,25 +69,29 @@ require ( github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-multistream v0.5.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/onsi/ginkgo/v2 v2.13.0 // indirect + github.com/onsi/ginkgo/v2 v2.15.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.17.0 // indirect - github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.11.1 // indirect - github.com/quic-go/qtls-go1-20 v0.3.4 // indirect - github.com/quic-go/quic-go v0.39.3 // indirect + github.com/prometheus/client_golang v1.19.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/quic-go/qtls-go1-20 v0.4.1 // indirect + github.com/quic-go/quic-go v0.40.1 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/supranational/blst v0.3.11 // indirect github.com/tyler-smith/go-bip39 v1.1.0 // indirect github.com/zeebo/errs v1.3.0 // indirect + go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.14.0 // indirect - golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect - golang.org/x/mod v0.13.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/tools v0.14.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 // indirect + golang.org/x/mod v0.15.0 // indirect + golang.org/x/net v0.22.0 // indirect + golang.org/x/sync v0.6.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/tools v0.17.0 // indirect + google.golang.org/protobuf v1.32.0 // indirect lukechampine.com/blake3 v1.2.1 // indirect + rsc.io/tmplfunc v0.0.3 // indirect storj.io/drpc v0.0.33 // indirect ) diff --git a/go.sum b/go.sum index 30d0342..4646894 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,19 @@ -filippo.io/edwards25519 v1.0.0 h1:0wAIcmJUqRdI8IJ/3eGi5/HwXZWPujYXXlkrQogz0Ek= -filippo.io/edwards25519 v1.0.0/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= +github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= +github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= +github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= +github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40= +github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= +github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/ahmetb/govvv v0.3.0 h1:YGLGwEyiUwHFy5eh/RUhdupbuaCGBYn5T5GWXp+WJB0= github.com/ahmetb/govvv v0.3.0/go.mod h1:4WRFpdWtc/YtKgPFwa1dr5+9hiRY5uKAL08bOlxOR6s= -github.com/anyproto/any-sync v0.3.7 h1:FR2Cx+3h6wxwB8NtPKtdtpG83eQk2zgjysaaJWlAyHk= -github.com/anyproto/any-sync v0.3.7/go.mod h1:EUxILCmnj9r+g08otHHwF0NgZVeisI+t6tPXVdWaPdE= +github.com/anyproto/any-sync v0.3.33 h1:ui7JRpeTbb8AiSyDfVWtrzCVgXg3/RnwWPhf21ROLS0= +github.com/anyproto/any-sync v0.3.33/go.mod h1:JORhDzUQfTRzvHfxOuHqLyyDYZdaeIBzVPafSkkSbSY= github.com/anyproto/go-chash v0.1.0 h1:I9meTPjXFRfXZHRJzjOHC/XF7Q5vzysKkiT/grsogXY= github.com/anyproto/go-chash v0.1.0/go.mod h1:0UjNQi3PDazP0fINpFYu6VKhuna+W/V+1vpXHAfNgLY= github.com/anyproto/go-slip10 v1.0.0 h1:uAEtSuudR3jJBOfkOXf3bErxVoxbuKwdoJN55M1i6IA= @@ -18,14 +25,55 @@ github.com/aws/aws-sdk-go v1.51.1/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3Tju github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bits-and-blooms/bitset v1.10.0 h1:ePXTeiPEazB5+opbv5fr8umg2R/1NlzgDsyepwsSr88= +github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btcd v0.22.1 h1:CnwP9LM/M9xuRrGSCGeMVs9iv09uMqwsVX7EeIpgV2c= +github.com/btcsuite/btcd v0.22.1/go.mod h1:wqgTSL29+50LRkmOVknEdmt8ZojIzhuWvgu/iptuN7Y= +github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= +github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce h1:YtWJF7RHm2pYCvA5t0RPmAaLUhREsKuKd+SLhxFbFeQ= +github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= +github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cheggaaa/mb/v3 v3.0.2 h1:jd1Xx0zzihZlXL6HmnRXVCI1BHuXz/kY+VzX9WbvNDU= +github.com/cheggaaa/mb/v3 v3.0.2/go.mod h1:zCt2QeYukhd/g0bIdNqF+b/kKz1hnLFNDkP49qN5kqI= +github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y= +github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 h1:aPEJyR4rPBvDmeyi+l/FS/VtA00IWvjeFvjen1m1l1A= +github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593/go.mod h1:6hk1eMY/u5t+Cf18q5lFMUA1Rc+Sm5I6Ra1QuPyxXCo= +github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw= +github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= +github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= +github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= +github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= +github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= +github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 h1:d28BXYi+wUpz1KBmiF9bWrjEMacUEREV6MBi2ODnrfQ= +github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= +github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA= +github.com/crate-crypto/go-kzg-4844 v0.7.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= +github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -37,8 +85,17 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etly github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R3nlY= +github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= +github.com/ethereum/go-ethereum v1.13.12 h1:iDr9UM2JWkngBHGovRJEQn4Kor7mT4gt9rUZqB5M29Y= +github.com/ethereum/go-ethereum v1.13.12/go.mod h1:hKL2Qcj1OvStXNSEDbucexqnEt1Wh4Cz329XsjAalZY= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 h1:BAIP2GihuqhwdILrV+7GJel5lyPV3u1+PgzrWLc0TkE= +github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46/go.mod h1:QNpY22eby74jVhqH4WhDLDwxc/vqsern6pW+u2kbkpc= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-redis/redis/v7 v7.4.0 h1:7obg6wUoj05T0EpY0o8B59S9w5yeMWql7sw2kwNW1x4= @@ -51,21 +108,22 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEe github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws= github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b h1:RMpPgZTSApbPf7xaVel+QkoGPRLFLrwFO89uDUHEGf0= -github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 h1:E/LAvt58di64hlYjx7AsNS6C/ysHWYo+2qPCZKTQhRo= +github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -73,6 +131,9 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= +github.com/holiman/uint256 v1.2.4 h1:jUc4Nk8fm9jZabQuqr2JzednajVmBpC+oiTiXZJEApU= +github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= @@ -83,12 +144,17 @@ github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -98,10 +164,14 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= +github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= -github.com/libp2p/go-libp2p v0.32.0 h1:86I4B7nBUPIyTgw3+5Ibq6K7DdKRCuZw8URCfPc1hQM= -github.com/libp2p/go-libp2p v0.32.0/go.mod h1:hXXC3kXPlBZ1eu8Q2hptGrMB4mZ3048JUoS4EKaHW5c= +github.com/libp2p/go-libp2p v0.32.2 h1:s8GYN4YJzgUoyeYNPdW7JZeZ5Ee31iNaIBfGYMAY4FQ= +github.com/libp2p/go-libp2p v0.32.2/go.mod h1:E0LKe+diV/ZVJVnOJby8VC5xzHF0660osg71skcxJvk= github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= github.com/libp2p/go-yamux/v4 v4.0.1 h1:FfDR4S1wj6Bw2Pqbc8Uz7pCxeRBPbwsBbEdfwiCypkQ= @@ -109,12 +179,17 @@ github.com/libp2p/go-yamux/v4 v4.0.1/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/miguelmota/go-ethereum-hdwallet v0.1.2 h1:mz9LO6V7QCRkLYb0AH17t5R8KeqCe3E+hx9YXpmZeXA= +github.com/miguelmota/go-ethereum-hdwallet v0.1.2/go.mod h1:fdNwFSoBFVBPnU0xpOd6l2ueqsPSH/Gch5kIvSvTGk8= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= +github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= +github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= @@ -136,31 +211,42 @@ github.com/multiformats/go-multistream v0.5.0/go.mod h1:n6tMZiwiP2wUsR8DgfDWw1dy github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= -github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= -github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY= +github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= -github.com/quic-go/qtls-go1-20 v0.3.4 h1:MfFAPULvst4yoMgY9QmtpYmfij/em7O8UUi+bNVm7Cg= -github.com/quic-go/qtls-go1-20 v0.3.4/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= -github.com/quic-go/quic-go v0.39.3 h1:o3YB6t2SR+HU/pgwF29kJ6g4jJIJEwEZ8CKia1h1TKg= -github.com/quic-go/quic-go v0.39.3/go.mod h1:T09QsDQWjLiQ74ZmacDfqZmhY/NLnw5BC40MANNNZ1Q= +github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= +github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/quic-go/qtls-go1-20 v0.4.1 h1:D33340mCNDAIKBqXuAvexTNMUByrYmFYVfKfDN5nfFs= +github.com/quic-go/qtls-go1-20 v0.4.1/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= +github.com/quic-go/quic-go v0.40.1 h1:X3AGzUNFs0jVuO3esAGnTfvdgvL4fq655WaOi1snv1Q= +github.com/quic-go/quic-go v0.40.1/go.mod h1:PeN7kuVJ4xZbxSv/4OX6S1USOX8MJvydwpTx31vx60c= github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/redis/rueidis v1.0.19 h1:s65oWtotzlIFN8eMPhyYwxlwLR1lUdhza2KtWprKYSo= github.com/redis/rueidis v1.0.19/go.mod h1:8B+r5wdnjwK3lTFml5VtxjzGOQAC+5UmujoD12pDrEo= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -172,6 +258,14 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 h1:QVqDTf3h2WHt08YuiTGPZLls0Wq99X9bWd0Q5ZSBesM= github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203/go.mod h1:oqN97ltKNihBbwlX8dLpwxCl3+HnXKV/R0e+sRLd9C8= +github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= +github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -195,36 +289,40 @@ go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8 go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 h1:/RIbNt/Zr7rVhIkQhooTxCxFcdWLGIKnZA4IXNFSrvo= +golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= -golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= -golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -235,33 +333,34 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= -golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= +golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -271,5 +370,7 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= +rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= +rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= storj.io/drpc v0.0.33 h1:yCGZ26r66ZdMP0IcTYsj7WDAUIIjzXk6DJhbhvt9FHI= storj.io/drpc v0.0.33/go.mod h1:vR804UNzhBa49NOJ6HeLjd2H3MakC1j5Gv8bsOQT6N4= diff --git a/index/bind.go b/index/bind.go index bf1cfb4..b04aa33 100644 --- a/index/bind.go +++ b/index/bind.go @@ -8,20 +8,18 @@ import ( ) func (ri *redisIndex) FileBind(ctx context.Context, key Key, fileId string, cids *CidEntries) (err error) { - var ( - sk = spaceKey(key) - gk = groupKey(key) - ) - _, gRelease, err := ri.AcquireKey(ctx, gk) - if err != nil { - return - } - defer gRelease() - _, sRelease, err := ri.AcquireKey(ctx, sk) + entry, release, err := ri.AcquireSpace(ctx, key) if err != nil { return } - defer sRelease() + defer release() + + return ri.fileBind(ctx, key, fileId, cids, entry) +} + +func (ri *redisIndex) fileBind(ctx context.Context, key Key, fileId string, cids *CidEntries, entry groupSpaceEntry) (err error) { + var gk = groupKey(key) + var sk = spaceKey(key) // get file entry fileInfo, isNewFile, err := ri.getFileEntry(ctx, key, fileId) @@ -29,6 +27,8 @@ func (ri *redisIndex) FileBind(ctx context.Context, key Key, fileId string, cids return } + isolatedSpace := entry.space.Limit != 0 + // make a list of indexes of non-exists cids var newFileCidIdx = make([]int, 0, len(cids.entries)) for i, c := range cids.entries { @@ -44,6 +44,8 @@ func (ri *redisIndex) FileBind(ctx context.Context, key Key, fileId string, cids return } + var affectedCidIdx = make([]int, 0, len(cids.entries)) + // get all cids from space and group in one pipeline var ( cidExistSpaceCmds = make([]*redis.BoolCmd, len(newFileCidIdx)) @@ -53,7 +55,9 @@ func (ri *redisIndex) FileBind(ctx context.Context, key Key, fileId string, cids for i, idx := range newFileCidIdx { ck := cidKey(cids.entries[idx].Cid) cidExistSpaceCmds[i] = pipe.HExists(ctx, sk, ck) - cidExistGroupCmds[i] = pipe.HExists(ctx, gk, ck) + if !isolatedSpace { + cidExistGroupCmds[i] = pipe.HExists(ctx, gk, ck) + } } return nil }) @@ -61,38 +65,31 @@ func (ri *redisIndex) FileBind(ctx context.Context, key Key, fileId string, cids return } - // load group and space info - spaceInfo, err := ri.getSpaceEntry(ctx, key) - if err != nil { - return - } - groupInfo, err := ri.getGroupEntry(ctx, key) - if err != nil { - return - } - // calculate new group and space stats for i, idx := range newFileCidIdx { - ex, err := cidExistGroupCmds[i].Result() - if err != nil { - return err - } - if !ex { - spaceInfo.CidCount++ - spaceInfo.Size_ += cids.entries[idx].Size_ + if !isolatedSpace { + ex, err := cidExistGroupCmds[i].Result() + if err != nil { + return err + } + if !ex { + entry.group.CidCount++ + entry.group.Size_ += cids.entries[idx].Size_ + } } - ex, err = cidExistSpaceCmds[i].Result() + ex, err := cidExistSpaceCmds[i].Result() if err != nil { return err } if !ex { - groupInfo.CidCount++ - groupInfo.Size_ += cids.entries[idx].Size_ + entry.space.CidCount++ + entry.space.Size_ += cids.entries[idx].Size_ + affectedCidIdx = append(affectedCidIdx, idx) } } - groupInfo.AddSpaceId(key.SpaceId) + entry.group.AddSpaceId(key.SpaceId) if isNewFile { - spaceInfo.FileCount++ + entry.space.FileCount++ } // make group and space updates in one tx @@ -100,18 +97,20 @@ func (ri *redisIndex) FileBind(ctx context.Context, key Key, fileId string, cids // increment cid refs for _, idx := range newFileCidIdx { ck := cidKey(cids.entries[idx].Cid) - tx.HIncrBy(ctx, gk, ck, 1) + if !isolatedSpace { + tx.HIncrBy(ctx, gk, ck, 1) + } tx.HIncrBy(ctx, sk, ck, 1) } // save info - spaceInfo.Save(ctx, key, tx) - groupInfo.Save(ctx, key, tx) + entry.space.Save(ctx, key, tx) + entry.group.Save(ctx, tx) fileInfo.Save(ctx, key, fileId, tx) return nil }) // update cids - for _, idx := range newFileCidIdx { + for _, idx := range affectedCidIdx { cids.entries[idx].Refs++ if saveErr := cids.entries[idx].Save(ctx, ri.cl); saveErr != nil { log.WarnCtx(ctx, "unable to save cid info", zap.Error(saveErr), zap.String("cid", cids.entries[idx].Cid.String())) diff --git a/index/bind_test.go b/index/bind_test.go index 5d4fae3..74f16ee 100644 --- a/index/bind_test.go +++ b/index/bind_test.go @@ -66,9 +66,11 @@ func TestRedisIndex_Bind(t *testing.T) { groupInfo, err := fx.GroupInfo(ctx, key.GroupId) require.NoError(t, err) assert.Equal(t, GroupInfo{ - BytesUsage: sumSize, - CidsCount: uint64(len(bs)), - SpaceIds: []string{key.SpaceId}, + BytesUsage: sumSize, + CidsCount: uint64(len(bs)), + AccountLimit: fx.defaultLimit, + Limit: fx.defaultLimit, + SpaceIds: []string{key.SpaceId}, }, groupInfo) }) diff --git a/index/cids.go b/index/cids.go index 10e84ba..5f38397 100644 --- a/index/cids.go +++ b/index/cids.go @@ -42,6 +42,7 @@ func (ri *redisIndex) CidEntriesByString(ctx context.Context, cids []string) (en return } if err = ri.getAndAddToEntries(ctx, entries, c); err != nil { + entries.Release() return nil, err } } diff --git a/index/delete.go b/index/delete.go index ddd580b..9c9b7ef 100644 --- a/index/delete.go +++ b/index/delete.go @@ -9,25 +9,20 @@ import ( ) func (ri *redisIndex) SpaceDelete(ctx context.Context, key Key) (ok bool, err error) { - var ( - gk = groupKey(key) - sk = spaceKey(key) - ) - - _, gRelease, err := ri.AcquireKey(ctx, gk) - if err != nil { - return - } - defer gRelease() - spaceExists, sRelease, err := ri.AcquireKey(ctx, sk) + entry, release, err := ri.AcquireSpace(ctx, key) if err != nil { return } - defer sRelease() + defer release() + + return ri.spaceDelete(ctx, key, entry) +} - if !spaceExists { +func (ri *redisIndex) spaceDelete(ctx context.Context, key Key, entry groupSpaceEntry) (ok bool, err error) { + if !entry.spaceExists { return false, nil } + sk := spaceKey(key) keys, err := ri.cl.HKeys(ctx, sk).Result() if err != nil { @@ -35,25 +30,27 @@ func (ri *redisIndex) SpaceDelete(ctx context.Context, key Key) (ok bool, err er } for _, k := range keys { if strings.HasPrefix(k, "f:") { - if err = ri.fileUnbind(ctx, key, k[2:]); err != nil { + if err = ri.fileUnbind(ctx, key, entry, k[2:]); err != nil { return } } } - groupInfo, err := ri.getGroupEntry(ctx, key) - if err != nil { - return - } - if !slices.Contains(groupInfo.SpaceIds, key.SpaceId) { + if !slices.Contains(entry.group.SpaceIds, key.SpaceId) { return false, nil } - groupInfo.SpaceIds = slices.DeleteFunc(groupInfo.SpaceIds, func(spaceId string) bool { + entry.group.SpaceIds = slices.DeleteFunc(entry.group.SpaceIds, func(spaceId string) bool { return spaceId == key.SpaceId }) + // in case of isolated space - return limit to the group + if entry.space.Limit != 0 { + entry.group.Limit += entry.space.Limit + } + _, err = ri.cl.Pipelined(ctx, func(pipe redis.Pipeliner) error { - groupInfo.Save(ctx, key, pipe) + entry.group.Save(ctx, pipe) + pipe.Del(ctx, sk) return nil }) if err != nil { diff --git a/index/delete_test.go b/index/delete_test.go index ce6a165..e047e38 100644 --- a/index/delete_test.go +++ b/index/delete_test.go @@ -45,6 +45,9 @@ func TestRedisIndex_SpaceDelete(t *testing.T) { assert.NotEmpty(t, groupInfo.BytesUsage) assert.Contains(t, groupInfo.SpaceIds, key.SpaceId) + require.NoError(t, fx.SetGroupLimit(ctx, key.GroupId, 5000)) + require.NoError(t, fx.SetSpaceLimit(ctx, key, 4000)) + ok, err = fx.SpaceDelete(ctx, key) require.NoError(t, err) assert.True(t, ok) @@ -53,6 +56,8 @@ func TestRedisIndex_SpaceDelete(t *testing.T) { require.NoError(t, err) assert.Empty(t, groupInfo.BytesUsage) assert.NotContains(t, groupInfo.SpaceIds, key.SpaceId) + assert.Equal(t, uint64(5000), groupInfo.AccountLimit) + assert.Equal(t, uint64(5000), groupInfo.Limit) // second call ok, err = fx.SpaceDelete(ctx, key) diff --git a/index/entry.go b/index/entry.go index 835f329..6f73f77 100644 --- a/index/entry.go +++ b/index/entry.go @@ -48,6 +48,7 @@ func (ri *redisIndex) getFileEntry(ctx context.Context, k Key, fileId string) (e } type spaceEntry struct { + Id string *indexproto.SpaceEntry } @@ -68,6 +69,7 @@ func (ri *redisIndex) getSpaceEntry(ctx context.Context, key Key) (entry *spaceE if errors.Is(err, redis.Nil) { now := time.Now().Unix() return &spaceEntry{ + Id: key.SpaceId, SpaceEntry: &indexproto.SpaceEntry{ CreateTime: now, UpdateTime: now, @@ -79,20 +81,20 @@ func (ri *redisIndex) getSpaceEntry(ctx context.Context, key Key) (entry *spaceE if err = spaceEntryProto.Unmarshal([]byte(result)); err != nil { return } - return &spaceEntry{SpaceEntry: spaceEntryProto}, nil + return &spaceEntry{SpaceEntry: spaceEntryProto, Id: key.SpaceId}, nil } type groupEntry struct { *indexproto.GroupEntry } -func (f *groupEntry) Save(ctx context.Context, k Key, cl redis.Cmdable) { +func (f *groupEntry) Save(ctx context.Context, cl redis.Cmdable) { f.UpdateTime = time.Now().Unix() data, err := f.Marshal() if err != nil { return } - cl.HSet(ctx, groupKey(k), infoKey, data) + cl.HSet(ctx, groupKey(Key{GroupId: f.GroupId}), infoKey, data) } func (f *groupEntry) AddSpaceId(spaceId string) { @@ -110,8 +112,12 @@ func (ri *redisIndex) getGroupEntry(ctx context.Context, key Key) (entry *groupE now := time.Now().Unix() return &groupEntry{ GroupEntry: &indexproto.GroupEntry{ - CreateTime: now, - UpdateTime: now, + GroupId: key.GroupId, + CreateTime: now, + UpdateTime: now, + Size_: 0, + Limit: ri.defaultLimit, + AccountLimit: ri.defaultLimit, }, }, nil } @@ -119,5 +125,13 @@ func (ri *redisIndex) getGroupEntry(ctx context.Context, key Key) (entry *groupE if err = groupEntryProto.Unmarshal([]byte(result)); err != nil { return } + groupEntryProto.GroupId = key.GroupId return &groupEntry{GroupEntry: groupEntryProto}, nil } + +type groupSpaceEntry struct { + space *spaceEntry + group *groupEntry + spaceExists bool + groupExists bool +} diff --git a/index/index.go b/index/index.go index c8dfb2b..64cbb6e 100644 --- a/index/index.go +++ b/index/index.go @@ -5,6 +5,8 @@ import ( "context" "errors" "strconv" + "strings" + "sync" "time" "github.com/OneOfOne/xxhash" @@ -17,6 +19,7 @@ import ( "github.com/ipfs/go-cid" "github.com/redis/go-redis/v9" + "github.com/anyproto/any-sync-filenode/config" "github.com/anyproto/any-sync-filenode/redisprovider" "github.com/anyproto/any-sync-filenode/store/s3store" ) @@ -33,6 +36,7 @@ type Index interface { FileBind(ctx context.Context, key Key, fileId string, cidEntries *CidEntries) (err error) FileUnbind(ctx context.Context, kye Key, fileIds ...string) (err error) FileInfo(ctx context.Context, key Key, fileIds ...string) (fileInfo []FileInfo, err error) + FilesList(ctx context.Context, key Key) (fileIds []string, err error) GroupInfo(ctx context.Context, groupId string) (info GroupInfo, err error) SpaceInfo(ctx context.Context, key Key) (info SpaceInfo, err error) @@ -40,11 +44,17 @@ type Index interface { BlocksGetNonExistent(ctx context.Context, bs []blocks.Block) (nonExistent []blocks.Block, err error) BlocksLock(ctx context.Context, bs []blocks.Block) (unlock func(), err error) BlocksAdd(ctx context.Context, bs []blocks.Block) (err error) + OnBlockUploaded(ctx context.Context, bs ...blocks.Block) + WaitCidExists(ctx context.Context, c cid.Cid) (err error) CidExists(ctx context.Context, c cid.Cid) (ok bool, err error) CidEntries(ctx context.Context, cids []cid.Cid) (entries *CidEntries, err error) CidEntriesByBlocks(ctx context.Context, bs []blocks.Block) (entries *CidEntries, err error) - CidExistsInSpace(ctx context.Context, k Key, cids []cid.Cid) (exists []cid.Cid, err error) + CidExistsInSpace(ctx context.Context, key Key, cids []cid.Cid) (exists []cid.Cid, err error) + + SetGroupLimit(ctx context.Context, groupId string, limit uint64) (err error) + SetSpaceLimit(ctx context.Context, key Key, limit uint64) (err error) + CheckLimits(ctx context.Context, key Key) error Migrate(ctx context.Context, key Key) error @@ -62,14 +72,17 @@ type Key struct { } type GroupInfo struct { - BytesUsage uint64 - CidsCount uint64 - SpaceIds []string + BytesUsage uint64 + CidsCount uint64 + AccountLimit uint64 + Limit uint64 + SpaceIds []string } type SpaceInfo struct { BytesUsage uint64 CidsCount uint64 + Limit uint64 FileCount uint32 } @@ -101,14 +114,31 @@ type redisIndex struct { persistStore persistentStore persistTtl time.Duration ticker periodicsync.PeriodicSync + defaultLimit uint64 + + cidSubscriptionsMu sync.Mutex + cidSubscriptions map[string]map[chan struct{}]struct{} + + ctx context.Context + ctxCancel context.CancelFunc } func (ri *redisIndex) Init(a *app.App) (err error) { ri.cl = a.MustComponent(redisprovider.CName).(redisprovider.RedisProvider).Redis() ri.persistStore = a.MustComponent(s3store.CName).(persistentStore) ri.redsync = redsync.New(goredis.NewPool(ri.cl)) - // todo: move to config - ri.persistTtl = time.Hour + conf := app.MustComponent[*config.Config](a) + + ri.persistTtl = time.Second * time.Duration(conf.PersistTtl) + if ri.persistTtl == 0 { + ri.persistTtl = time.Hour + } + ri.defaultLimit = conf.DefaultLimit + if ri.defaultLimit == 0 { + ri.defaultLimit = 1 << 30 + } + ri.cidSubscriptions = make(map[string]map[chan struct{}]struct{}) + ri.ctx, ri.ctxCancel = context.WithCancel(context.Background()) return } @@ -122,6 +152,7 @@ func (ri *redisIndex) Run(ctx context.Context) (err error) { return nil }, log) ri.ticker.Run() + go ri.subscription(ctx) return } @@ -145,6 +176,25 @@ func (ri *redisIndex) FileInfo(ctx context.Context, key Key, fileIds ...string) return } +func (ri *redisIndex) FilesList(ctx context.Context, key Key) (fileIds []string, err error) { + sk := spaceKey(key) + _, release, err := ri.AcquireKey(ctx, sk) + if err != nil { + return + } + defer release() + allKeys, err := ri.cl.HKeys(ctx, sk).Result() + if err != nil { + return + } + for _, k := range allKeys { + if strings.HasPrefix(k, "f:") { + fileIds = append(fileIds, k[2:]) + } + } + return +} + func (ri *redisIndex) BlocksGetNonExistent(ctx context.Context, bs []blocks.Block) (nonExistent []blocks.Block, err error) { for _, b := range bs { ex, err := ri.CheckKey(ctx, cidKey(b.Cid())) @@ -189,9 +239,11 @@ func (ri *redisIndex) GroupInfo(ctx context.Context, groupId string) (info Group return } return GroupInfo{ - BytesUsage: sEntry.Size_, - CidsCount: sEntry.CidCount, - SpaceIds: sEntry.SpaceIds, + BytesUsage: sEntry.Size_, + CidsCount: sEntry.CidCount, + AccountLimit: sEntry.AccountLimit, + Limit: sEntry.Limit, + SpaceIds: sEntry.SpaceIds, }, nil } @@ -208,6 +260,7 @@ func (ri *redisIndex) SpaceInfo(ctx context.Context, key Key) (info SpaceInfo, e return SpaceInfo{ BytesUsage: sEntry.Size_, CidsCount: sEntry.CidCount, + Limit: sEntry.Limit, FileCount: sEntry.FileCount, }, nil } @@ -216,6 +269,9 @@ func (ri *redisIndex) Close(ctx context.Context) error { if ri.ticker != nil { ri.ticker.Close() } + if ri.ctxCancel != nil { + ri.ctxCancel() + } return nil } diff --git a/index/index_test.go b/index/index_test.go index a8f8b5e..0ef9fd6 100644 --- a/index/index_test.go +++ b/index/index_test.go @@ -5,9 +5,11 @@ import ( "testing" "github.com/anyproto/any-sync/app" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + "github.com/anyproto/any-sync-filenode/config" "github.com/anyproto/any-sync-filenode/redisprovider/testredisprovider" "github.com/anyproto/any-sync-filenode/store/mock_store" "github.com/anyproto/any-sync-filenode/store/s3store" @@ -16,6 +18,29 @@ import ( var ctx = context.Background() +func TestRedisIndex_FilesList(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + + k := newRandKey() + limit := uint64(10) + require.NoError(t, fx.SetSpaceLimit(ctx, k, limit)) + // no error + assert.NoError(t, fx.CheckLimits(ctx, k)) + + bs := testutil.NewRandBlocks(3) + require.NoError(t, fx.BlocksAdd(ctx, bs)) + fileId := testutil.NewRandCid().String() + cids, err := fx.CidEntriesByBlocks(ctx, bs) + require.NoError(t, err) + require.NoError(t, fx.FileBind(ctx, k, fileId, cids)) + cids.Release() + + fileIds, err := fx.FilesList(ctx, k) + require.NoError(t, err) + assert.Equal(t, []string{fileId}, fileIds) +} + func newRandKey() Key { return Key{ SpaceId: testutil.NewRandSpaceId(), @@ -24,6 +49,10 @@ func newRandKey() Key { } func newFixture(t *testing.T) (fx *fixture) { + return newFixtureConfig(t, nil) +} + +func newFixtureConfig(t *testing.T, conf *config.Config) (fx *fixture) { ctrl := gomock.NewController(t) fx = &fixture{ redisIndex: New().(*redisIndex), @@ -33,8 +62,10 @@ func newFixture(t *testing.T) (fx *fixture) { } fx.persistStore.EXPECT().Name().Return(s3store.CName).AnyTimes() fx.persistStore.EXPECT().Init(gomock.Any()).AnyTimes() - - fx.a.Register(testredisprovider.NewTestRedisProvider()).Register(fx.redisIndex).Register(fx.persistStore) + if conf == nil { + conf = &config.Config{DefaultLimit: 1024, PersistTtl: 3600} + } + fx.a.Register(testredisprovider.NewTestRedisProvider()).Register(fx.redisIndex).Register(fx.persistStore).Register(conf) require.NoError(t, fx.a.Start(ctx)) return } diff --git a/index/indexproto/index.pb.go b/index/indexproto/index.pb.go index 80851f0..d5e0890 100644 --- a/index/indexproto/index.pb.go +++ b/index/indexproto/index.pb.go @@ -143,12 +143,14 @@ func (m *CidList) GetCids() [][]byte { } type GroupEntry struct { - GroupId string `protobuf:"bytes,1,opt,name=groupId,proto3" json:"groupId,omitempty"` - CreateTime int64 `protobuf:"varint,2,opt,name=createTime,proto3" json:"createTime,omitempty"` - UpdateTime int64 `protobuf:"varint,3,opt,name=updateTime,proto3" json:"updateTime,omitempty"` - Size_ uint64 `protobuf:"varint,4,opt,name=size,proto3" json:"size,omitempty"` - CidCount uint64 `protobuf:"varint,5,opt,name=cidCount,proto3" json:"cidCount,omitempty"` - SpaceIds []string `protobuf:"bytes,6,rep,name=spaceIds,proto3" json:"spaceIds,omitempty"` + GroupId string `protobuf:"bytes,1,opt,name=groupId,proto3" json:"groupId,omitempty"` + CreateTime int64 `protobuf:"varint,2,opt,name=createTime,proto3" json:"createTime,omitempty"` + UpdateTime int64 `protobuf:"varint,3,opt,name=updateTime,proto3" json:"updateTime,omitempty"` + Size_ uint64 `protobuf:"varint,4,opt,name=size,proto3" json:"size,omitempty"` + CidCount uint64 `protobuf:"varint,5,opt,name=cidCount,proto3" json:"cidCount,omitempty"` + SpaceIds []string `protobuf:"bytes,6,rep,name=spaceIds,proto3" json:"spaceIds,omitempty"` + Limit uint64 `protobuf:"varint,7,opt,name=limit,proto3" json:"limit,omitempty"` + AccountLimit uint64 `protobuf:"varint,8,opt,name=accountLimit,proto3" json:"accountLimit,omitempty"` } func (m *GroupEntry) Reset() { *m = GroupEntry{} } @@ -226,6 +228,20 @@ func (m *GroupEntry) GetSpaceIds() []string { return nil } +func (m *GroupEntry) GetLimit() uint64 { + if m != nil { + return m.Limit + } + return 0 +} + +func (m *GroupEntry) GetAccountLimit() uint64 { + if m != nil { + return m.AccountLimit + } + return 0 +} + type SpaceEntry struct { GroupId string `protobuf:"bytes,1,opt,name=groupId,proto3" json:"groupId,omitempty"` CreateTime int64 `protobuf:"varint,2,opt,name=createTime,proto3" json:"createTime,omitempty"` @@ -233,6 +249,7 @@ type SpaceEntry struct { Size_ uint64 `protobuf:"varint,4,opt,name=size,proto3" json:"size,omitempty"` FileCount uint32 `protobuf:"varint,5,opt,name=fileCount,proto3" json:"fileCount,omitempty"` CidCount uint64 `protobuf:"varint,6,opt,name=cidCount,proto3" json:"cidCount,omitempty"` + Limit uint64 `protobuf:"varint,7,opt,name=limit,proto3" json:"limit,omitempty"` } func (m *SpaceEntry) Reset() { *m = SpaceEntry{} } @@ -310,6 +327,13 @@ func (m *SpaceEntry) GetCidCount() uint64 { return 0 } +func (m *SpaceEntry) GetLimit() uint64 { + if m != nil { + return m.Limit + } + return 0 +} + type FileEntry struct { Cids []string `protobuf:"bytes,1,rep,name=cids,proto3" json:"cids,omitempty"` Size_ uint64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` @@ -391,29 +415,31 @@ func init() { } var fileDescriptor_f1f29953df8d243b = []byte{ - // 342 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x92, 0xcf, 0x4a, 0xc3, 0x40, - 0x10, 0xc6, 0x3b, 0x4d, 0xfa, 0x27, 0x83, 0x15, 0xc9, 0x69, 0x11, 0x5d, 0x42, 0xbc, 0x2c, 0x1e, - 0xec, 0xc1, 0x37, 0xb0, 0xa8, 0x14, 0x3c, 0xc8, 0xea, 0xc9, 0x5b, 0xcd, 0x6e, 0x65, 0xa1, 0x26, - 0x61, 0x37, 0x15, 0xf5, 0x19, 0x3c, 0xf8, 0x2c, 0xe2, 0x43, 0x78, 0xec, 0xd1, 0xa3, 0xb4, 0x2f, - 0x22, 0xbb, 0xa1, 0x49, 0x5b, 0x10, 0x2f, 0x82, 0x97, 0xcd, 0xfc, 0xbe, 0x21, 0xc9, 0xf7, 0xed, - 0x0c, 0x1e, 0xa8, 0x54, 0xc8, 0xc7, 0xbe, 0x3b, 0x73, 0x9d, 0x15, 0x59, 0xdf, 0x9d, 0xa6, 0x54, - 0x8e, 0x1c, 0x84, 0xdb, 0x63, 0x35, 0x91, 0x43, 0x2b, 0x5c, 0x5a, 0x8e, 0x5f, 0x00, 0xbb, 0x03, - 0x25, 0x4e, 0xd3, 0x42, 0x3f, 0x85, 0x21, 0xfa, 0x46, 0x3d, 0x4b, 0x02, 0x11, 0x30, 0x9f, 0xbb, - 0x3a, 0xa4, 0x88, 0x89, 0x96, 0xa3, 0x42, 0x5e, 0xab, 0x7b, 0x49, 0x9a, 0x11, 0x30, 0x8f, 0xaf, - 0x28, 0xb6, 0x3f, 0xcd, 0xc5, 0xb2, 0xef, 0x95, 0xfd, 0x5a, 0xb1, 0xdf, 0xd4, 0x72, 0x6c, 0x88, - 0x1f, 0x01, 0x6b, 0x71, 0x57, 0x87, 0x04, 0x3b, 0x0f, 0x52, 0x1b, 0x95, 0xa5, 0xa4, 0x15, 0x01, - 0xeb, 0xf1, 0x25, 0xc6, 0xfb, 0xd8, 0x19, 0x28, 0x71, 0xa1, 0x4c, 0x61, 0x5f, 0x4c, 0x94, 0x30, - 0x04, 0x22, 0x8f, 0x6d, 0x71, 0x57, 0xc7, 0x6f, 0x80, 0x78, 0xae, 0xb3, 0x69, 0x5e, 0xfa, 0x25, - 0xd8, 0xb9, 0xb3, 0x34, 0x14, 0xce, 0x72, 0xc0, 0x97, 0xf8, 0x17, 0xae, 0xdd, 0x4d, 0xf8, 0x2b, - 0x37, 0xb1, 0x8b, 0xdd, 0x44, 0x89, 0x41, 0x36, 0x4d, 0x0b, 0x67, 0xdb, 0xe7, 0x15, 0xdb, 0x9e, - 0xc9, 0x47, 0x89, 0x1c, 0x0a, 0x43, 0xda, 0x91, 0xc7, 0x02, 0x5e, 0x71, 0xfc, 0x0e, 0x88, 0x57, - 0x16, 0xfe, 0xc3, 0xf4, 0x1e, 0x06, 0x76, 0xe2, 0xb5, 0xeb, 0x1e, 0xaf, 0x85, 0xb5, 0x48, 0xed, - 0xf5, 0x48, 0xb1, 0xc1, 0xe0, 0x4c, 0x4d, 0x64, 0xb5, 0x19, 0xd5, 0x30, 0x82, 0x72, 0x18, 0xd5, - 0xef, 0x9a, 0x3f, 0x6e, 0x8b, 0xf7, 0x4b, 0x04, 0x7f, 0x33, 0xc2, 0xc9, 0xe1, 0xc7, 0x9c, 0xc2, - 0x6c, 0x4e, 0xe1, 0x6b, 0x4e, 0xe1, 0x75, 0x41, 0x1b, 0xb3, 0x05, 0x6d, 0x7c, 0x2e, 0x68, 0xe3, - 0x66, 0x67, 0x73, 0xbb, 0x6f, 0xdb, 0xee, 0x71, 0xfc, 0x1d, 0x00, 0x00, 0xff, 0xff, 0xb9, 0xc4, - 0xb2, 0x7d, 0xf8, 0x02, 0x00, 0x00, + // 372 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x53, 0xcd, 0x4a, 0xf3, 0x40, + 0x14, 0xed, 0x34, 0x69, 0xd3, 0x5c, 0xda, 0x8f, 0x8f, 0xc1, 0xc5, 0x20, 0x3a, 0x84, 0xb8, 0x09, + 0x2e, 0xec, 0xc2, 0x37, 0xb0, 0xa8, 0x14, 0xba, 0x90, 0xd1, 0x95, 0xbb, 0x9a, 0x99, 0xca, 0x40, + 0x9b, 0x84, 0x4c, 0x2a, 0xea, 0x33, 0xb8, 0xf0, 0xb1, 0xc4, 0x55, 0x97, 0x2e, 0xa5, 0xdd, 0xfa, + 0x10, 0x32, 0x37, 0x34, 0xfd, 0x41, 0x71, 0x23, 0xb8, 0x99, 0xdc, 0x73, 0x0e, 0x77, 0x38, 0xe7, + 0xce, 0x0d, 0x1c, 0xe8, 0x44, 0xaa, 0xfb, 0x2e, 0x9e, 0x59, 0x9e, 0x16, 0x69, 0x17, 0x4f, 0x53, + 0x32, 0x47, 0x08, 0xe8, 0xbf, 0x91, 0x1e, 0xab, 0xbe, 0x25, 0x2e, 0x2c, 0x0e, 0x9f, 0x08, 0xb4, + 0x7a, 0x5a, 0x9e, 0x26, 0x45, 0xfe, 0x40, 0x29, 0xb8, 0x46, 0x3f, 0x2a, 0x46, 0x02, 0x12, 0xb9, + 0x02, 0x6b, 0xca, 0x01, 0xe2, 0x5c, 0x0d, 0x0b, 0x75, 0xa5, 0x27, 0x8a, 0xd5, 0x03, 0x12, 0x39, + 0x62, 0x8d, 0xb1, 0xfa, 0x34, 0x93, 0x4b, 0xdd, 0x29, 0xf5, 0x15, 0x63, 0xef, 0xcc, 0xd5, 0xc8, + 0x30, 0x37, 0x20, 0x51, 0x43, 0x60, 0x4d, 0x19, 0x78, 0x77, 0x2a, 0x37, 0x3a, 0x4d, 0x58, 0x23, + 0x20, 0x51, 0x47, 0x2c, 0x61, 0xb8, 0x0f, 0x5e, 0x4f, 0xcb, 0x81, 0x36, 0x85, 0x6d, 0x8c, 0xb5, + 0x34, 0x8c, 0x04, 0x4e, 0xd4, 0x16, 0x58, 0x87, 0x1f, 0x04, 0xe0, 0x3c, 0x4f, 0xa7, 0x59, 0xe9, + 0x97, 0x81, 0x77, 0x6b, 0x51, 0x5f, 0xa2, 0x65, 0x5f, 0x2c, 0xe1, 0x6f, 0xb8, 0xc6, 0x49, 0xb8, + 0x6b, 0x93, 0xd8, 0x85, 0x56, 0xac, 0x65, 0x2f, 0x9d, 0x26, 0x05, 0xda, 0x76, 0x45, 0x85, 0xad, + 0x66, 0xb2, 0x61, 0xac, 0xfa, 0xd2, 0xb0, 0x66, 0xe0, 0x44, 0xbe, 0xa8, 0x30, 0xdd, 0x81, 0xc6, + 0x58, 0x4f, 0x74, 0xc1, 0x3c, 0x6c, 0x2a, 0x01, 0x0d, 0xa1, 0x3d, 0x8c, 0x63, 0xdb, 0x3c, 0x40, + 0xb1, 0x85, 0xe2, 0x06, 0x17, 0xbe, 0x12, 0x80, 0x4b, 0x7b, 0xcd, 0x5f, 0xc4, 0xdd, 0x03, 0xdf, + 0xee, 0xca, 0x2a, 0x6f, 0x47, 0xac, 0x88, 0x8d, 0x61, 0x34, 0xb7, 0x86, 0xf1, 0x65, 0xe0, 0xd0, + 0x80, 0x7f, 0xa6, 0xc7, 0xaa, 0xda, 0xb4, 0xea, 0x71, 0xfd, 0xf2, 0x71, 0x2b, 0x13, 0xf5, 0x6f, + 0xb7, 0xcf, 0xf9, 0x21, 0x98, 0xbb, 0x1d, 0xec, 0xe4, 0xf0, 0x65, 0xce, 0xc9, 0x6c, 0xce, 0xc9, + 0xfb, 0x9c, 0x93, 0xe7, 0x05, 0xaf, 0xcd, 0x16, 0xbc, 0xf6, 0xb6, 0xe0, 0xb5, 0xeb, 0xff, 0xdb, + 0x7f, 0xcb, 0x4d, 0x13, 0x3f, 0xc7, 0x9f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x64, 0x5b, 0xf6, 0x08, + 0x48, 0x03, 0x00, 0x00, } func (m *CidEntry) Marshal() (dAtA []byte, err error) { @@ -516,6 +542,16 @@ func (m *GroupEntry) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.AccountLimit != 0 { + i = encodeVarintIndex(dAtA, i, uint64(m.AccountLimit)) + i-- + dAtA[i] = 0x40 + } + if m.Limit != 0 { + i = encodeVarintIndex(dAtA, i, uint64(m.Limit)) + i-- + dAtA[i] = 0x38 + } if len(m.SpaceIds) > 0 { for iNdEx := len(m.SpaceIds) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.SpaceIds[iNdEx]) @@ -575,6 +611,11 @@ func (m *SpaceEntry) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Limit != 0 { + i = encodeVarintIndex(dAtA, i, uint64(m.Limit)) + i-- + dAtA[i] = 0x38 + } if m.CidCount != 0 { i = encodeVarintIndex(dAtA, i, uint64(m.CidCount)) i-- @@ -735,6 +776,12 @@ func (m *GroupEntry) Size() (n int) { n += 1 + l + sovIndex(uint64(l)) } } + if m.Limit != 0 { + n += 1 + sovIndex(uint64(m.Limit)) + } + if m.AccountLimit != 0 { + n += 1 + sovIndex(uint64(m.AccountLimit)) + } return n } @@ -763,6 +810,9 @@ func (m *SpaceEntry) Size() (n int) { if m.CidCount != 0 { n += 1 + sovIndex(uint64(m.CidCount)) } + if m.Limit != 0 { + n += 1 + sovIndex(uint64(m.Limit)) + } return n } @@ -1192,6 +1242,44 @@ func (m *GroupEntry) Unmarshal(dAtA []byte) error { } m.SpaceIds = append(m.SpaceIds, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Limit", wireType) + } + m.Limit = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Limit |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 8: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AccountLimit", wireType) + } + m.AccountLimit = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AccountLimit |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipIndex(dAtA[iNdEx:]) @@ -1369,6 +1457,25 @@ func (m *SpaceEntry) Unmarshal(dAtA []byte) error { break } } + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Limit", wireType) + } + m.Limit = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Limit |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipIndex(dAtA[iNdEx:]) diff --git a/index/indexproto/protos/index.proto b/index/indexproto/protos/index.proto index e220e2a..b4477e0 100644 --- a/index/indexproto/protos/index.proto +++ b/index/indexproto/protos/index.proto @@ -22,6 +22,8 @@ message GroupEntry { uint64 size = 4; uint64 cidCount = 5; repeated string spaceIds = 6; + uint64 limit = 7; + uint64 accountLimit = 8; } message SpaceEntry { @@ -31,6 +33,7 @@ message SpaceEntry { uint64 size = 4; uint32 fileCount = 5; uint64 cidCount = 6; + uint64 limit = 7; } message FileEntry { diff --git a/index/limit.go b/index/limit.go new file mode 100644 index 0000000..e17fa12 --- /dev/null +++ b/index/limit.go @@ -0,0 +1,313 @@ +package index + +import ( + "context" + "errors" + "strconv" + "strings" + + "github.com/anyproto/any-sync/commonfile/fileproto/fileprotoerr" + "github.com/ipfs/go-cid" + "github.com/redis/go-redis/v9" + "go.uber.org/zap" +) + +var ErrLimitExceed = errors.New("limit exceed") + +func (ri *redisIndex) CheckLimits(ctx context.Context, key Key) (err error) { + entry, release, err := ri.AcquireSpace(ctx, key) + if err != nil { + return + } + defer release() + + // isolated space + if entry.space.Limit != 0 { + if entry.space.Size_ >= entry.space.Limit { + return ErrLimitExceed + } + return + } + + // group limit + if entry.group.Size_ >= entry.group.Limit { + return ErrLimitExceed + } + return +} + +func (ri *redisIndex) SetGroupLimit(ctx context.Context, groupId string, limit uint64) (err error) { + op := &spaceLimitOp{ + redisIndex: ri, + } + return op.SetGroupLimit(ctx, groupId, limit) +} + +func (ri *redisIndex) SetSpaceLimit(ctx context.Context, key Key, limit uint64) (err error) { + op := &spaceLimitOp{ + redisIndex: ri, + } + return op.SetSpaceLimit(ctx, key, limit) +} + +type spaceLimitOp struct { + *redisIndex + groupEntry *groupEntry + spaceEntries []*spaceEntry + release []func() +} + +func (op *spaceLimitOp) SetGroupLimit(ctx context.Context, groupId string, limit uint64) (err error) { + _, release, err := op.AcquireKey(ctx, groupKey(Key{GroupId: groupId})) + if err != nil { + return + } + op.release = append(op.release, release) + defer op.releaseAll() + + op.groupEntry, err = op.getGroupEntry(ctx, Key{GroupId: groupId}) + if err != nil { + return + } + + if op.groupEntry.AccountLimit == limit { + // same limit - do nothing + return + } + + isolatedLimit := op.groupEntry.AccountLimit - op.groupEntry.Limit + if op.groupEntry.AccountLimit > limit { + // limit is decreasing + // check the isolated limit, if the new limit is covering the isolated limit - do nothing + if isolatedLimit > limit { + // decrease the limit for isolated spaces in the same proportion as the account limit + isolatedLimit, err = op.decreaseIsolatedLimit(ctx, float64(limit)/float64(op.groupEntry.AccountLimit)) + if err != nil { + return err + } + } + } + + op.groupEntry.Limit = limit - isolatedLimit + op.groupEntry.AccountLimit = limit + return op.saveAll(ctx) +} + +func (op *spaceLimitOp) decreaseIsolatedLimit(ctx context.Context, k float64) (newIsolatedLimit uint64, err error) { + for _, spaceId := range op.groupEntry.SpaceIds { + newLimit, err := op.decreaseIsolatedLimitForSpace(ctx, spaceId, k) + if err != nil { + return 0, err + } + newIsolatedLimit += newLimit + } + return +} + +func (op *spaceLimitOp) decreaseIsolatedLimitForSpace(ctx context.Context, spaceId string, k float64) (newIsolatedLimit uint64, err error) { + key := Key{GroupId: op.groupEntry.GroupId, SpaceId: spaceId} + _, release, err := op.AcquireKey(ctx, spaceKey(key)) + if err != nil { + return + } + op.release = append(op.release, release) + + sEntry, err := op.getSpaceEntry(ctx, key) + if err != nil { + return + } + if sEntry.Limit == 0 { + // not isolated space + return + } + + sEntry.Limit = uint64(float64(sEntry.Limit) * k) + op.spaceEntries = append(op.spaceEntries, sEntry) + return sEntry.Limit, nil +} + +func (op *spaceLimitOp) SetSpaceLimit(ctx context.Context, key Key, limit uint64) (err error) { + entry, release, err := op.AcquireSpace(ctx, key) + if err != nil { + return + } + op.release = append(op.release, release) + defer op.releaseAll() + + op.groupEntry = entry.group + op.spaceEntries = append(op.spaceEntries, entry.space) + + if entry.space.Limit == limit { + // same limit - do nothing + return + } + prevLimit := entry.space.Limit + entry.space.Limit = limit + + if limit != 0 { + diff := int64(limit) - int64(prevLimit) + newGLimit := int64(op.groupEntry.Limit) - diff + + // check the group limit + if newGLimit < int64(op.groupEntry.Size_) { + return fileprotoerr.ErrNotEnoughSpace + } + + // check the space limit + if entry.space.Size_ > limit { + return fileprotoerr.ErrNotEnoughSpace + } + op.groupEntry.Limit = uint64(newGLimit) + + if prevLimit == 0 { + if err = op.isolateSpace(ctx, entry); err != nil { + return + } + } + } else { + if err = op.uniteSpace(ctx, entry); err != nil { + return + } + op.groupEntry.Limit += prevLimit + } + entry.group.AddSpaceId(key.SpaceId) + return op.saveAll(ctx) +} + +func (op *spaceLimitOp) isolateSpace(ctx context.Context, entry groupSpaceEntry) (err error) { + key := Key{GroupId: entry.group.GroupId, SpaceId: entry.space.Id} + sk := spaceKey(key) + gk := groupKey(key) + + keys, err := op.cl.HGetAll(ctx, sk).Result() + if err != nil { + return + } + + var cids []cid.Cid + var cidRefs []int64 + // collect all cids + for k, val := range keys { + if strings.HasPrefix(k, "c:") { + c, cErr := cid.Decode(k[2:]) + if cErr != nil { + log.WarnCtx(ctx, "can't decode cid", zap.String("cid", k[2:]), zap.Error(cErr)) + } else { + ref, _ := strconv.ParseInt(val, 10, 64) + cids = append(cids, c) + cidRefs = append(cidRefs, ref) + } + } + } + + // fetch cid entries + // TODO: we don't need to take a lock here, but for now it easiest way + cidEntries, err := op.CidEntries(ctx, cids) + if err != nil { + return err + } + defer cidEntries.Release() + + // decrement refs in the group + var groupDecrResults = make([]*redis.IntCmd, len(cids)) + if _, err = op.cl.TxPipelined(ctx, func(pipe redis.Pipeliner) error { + for i, c := range cids { + groupDecrResults[i] = pipe.HIncrBy(ctx, gk, cidKey(c), -cidRefs[i]) + } + return nil + }); err != nil { + return + } + + // make a list of cids without refs + var toDeleteIdx = make([]int, 0, len(cids)) + for i, res := range groupDecrResults { + if res.Val() <= 0 { + toDeleteIdx = append(toDeleteIdx, i) + } + } + + // remove cids without refs from the group and decrease the size + if len(toDeleteIdx) > 0 { + if _, err = op.cl.TxPipelined(ctx, func(pipe redis.Pipeliner) error { + for _, idx := range toDeleteIdx { + pipe.HDel(ctx, gk, cidKey(cids[idx])) + op.groupEntry.Size_ -= cidEntries.entries[idx].Size_ + op.groupEntry.CidCount-- + } + return nil + }); err != nil { + return err + } + } + return +} + +func (op *spaceLimitOp) uniteSpace(ctx context.Context, entry groupSpaceEntry) (err error) { + key := Key{GroupId: entry.group.GroupId, SpaceId: entry.space.Id} + sk := spaceKey(key) + gk := groupKey(key) + + keys, err := op.cl.HKeys(ctx, sk).Result() + if err != nil { + return + } + + var cids []cid.Cid + // collect all cids + for _, k := range keys { + if strings.HasPrefix(k, "c:") { + c, cErr := cid.Decode(k[2:]) + if cErr != nil { + log.WarnCtx(ctx, "can't decode cid", zap.String("cid", k[2:]), zap.Error(cErr)) + } else { + cids = append(cids, c) + } + } + } + + // fetch cid entries + // TODO: we don't need to take a lock here, but for now it easiest way + cidEntries, err := op.CidEntries(ctx, cids) + if err != nil { + return err + } + defer cidEntries.Release() + + // increment refs in the group + var groupDecrResults = make([]*redis.IntCmd, len(cids)) + if _, err = op.cl.TxPipelined(ctx, func(pipe redis.Pipeliner) error { + for i, c := range cids { + groupDecrResults[i] = pipe.HIncrBy(ctx, gk, cidKey(c), 1) + } + return nil + }); err != nil { + return + } + + // make a list of cids without refs and increase the size + for i, res := range groupDecrResults { + if res.Val() == 1 { + op.groupEntry.Size_ += cidEntries.entries[i].Size_ + op.groupEntry.CidCount++ + } + } + return +} + +func (op *spaceLimitOp) releaseAll() { + for _, r := range op.release { + r() + } +} + +func (op *spaceLimitOp) saveAll(ctx context.Context) (err error) { + _, err = op.cl.TxPipelined(ctx, func(tx redis.Pipeliner) error { + for _, sEntry := range op.spaceEntries { + sEntry.Save(ctx, Key{GroupId: sEntry.GroupId, SpaceId: sEntry.Id}, tx) + } + op.groupEntry.Save(ctx, tx) + return nil + }) + return +} diff --git a/index/limit_test.go b/index/limit_test.go new file mode 100644 index 0000000..fbf1c3d --- /dev/null +++ b/index/limit_test.go @@ -0,0 +1,262 @@ +package index + +import ( + "testing" + + "github.com/anyproto/any-sync/commonfile/fileproto/fileprotoerr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/any-sync-filenode/testutil" +) + +func TestRedisIndex_CheckLimits(t *testing.T) { + t.Run("isolated space", func(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + + k := newRandKey() + limit := uint64(10) + require.NoError(t, fx.SetSpaceLimit(ctx, k, limit)) + // no error + assert.NoError(t, fx.CheckLimits(ctx, k)) + + // add 10 blocks + bs := testutil.NewRandBlocks(10) + require.NoError(t, fx.BlocksAdd(ctx, bs)) + fileId := testutil.NewRandCid().String() + cids, err := fx.CidEntriesByBlocks(ctx, bs) + require.NoError(t, err) + require.NoError(t, fx.FileBind(ctx, k, fileId, cids)) + cids.Release() + + assert.ErrorIs(t, fx.CheckLimits(ctx, k), ErrLimitExceed) + }) + t.Run("group", func(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + + k := newRandKey() + limit := uint64(10) + require.NoError(t, fx.SetGroupLimit(ctx, k.GroupId, limit)) + // no error + assert.NoError(t, fx.CheckLimits(ctx, k)) + + // add 10 blocks + bs := testutil.NewRandBlocks(10) + require.NoError(t, fx.BlocksAdd(ctx, bs)) + fileId := testutil.NewRandCid().String() + cids, err := fx.CidEntriesByBlocks(ctx, bs) + require.NoError(t, err) + require.NoError(t, fx.FileBind(ctx, k, fileId, cids)) + cids.Release() + + assert.ErrorIs(t, fx.CheckLimits(ctx, k), ErrLimitExceed) + }) + +} + +func TestRedisIndex_SetGroupLimit(t *testing.T) { + t.Run("increase limit", func(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + + k := newRandKey() + limit := uint64(3456) + require.NoError(t, fx.SetGroupLimit(ctx, k.GroupId, limit)) + + info, err := fx.GroupInfo(ctx, k.GroupId) + require.NoError(t, err) + assert.Equal(t, limit, info.Limit) + assert.Equal(t, limit, info.AccountLimit) + }) + t.Run("decrease limit", func(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + + k := newRandKey() + limit := uint64(100) + require.NoError(t, fx.SetGroupLimit(ctx, k.GroupId, limit)) + + info, err := fx.GroupInfo(ctx, k.GroupId) + require.NoError(t, err) + assert.Equal(t, limit, info.Limit) + assert.Equal(t, limit, info.AccountLimit) + }) + t.Run("decrease limit with isolated spaces", func(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + + k := newRandKey() + initialLimit := uint64(3000) + require.NoError(t, fx.SetGroupLimit(ctx, k.GroupId, initialLimit)) + require.NoError(t, fx.SetSpaceLimit(ctx, k, 2000)) + + require.NoError(t, fx.SetGroupLimit(ctx, k.GroupId, 1000)) + + groupInfo, err := fx.GroupInfo(ctx, k.GroupId) + require.NoError(t, err) + assert.Equal(t, uint64(334), groupInfo.Limit) + assert.Equal(t, uint64(1000), groupInfo.AccountLimit) + + spaceInfo, err := fx.SpaceInfo(ctx, k) + require.NoError(t, err) + assert.Equal(t, uint64(666), spaceInfo.Limit) + t.Log(spaceInfo.Limit) + + }) + +} + +func TestRedisIndex_SetSpaceLimit(t *testing.T) { + t.Run("isolate empty space", func(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + + k := newRandKey() + groupLimit := uint64(3000) + spaceLimit := uint64(1000) + require.NoError(t, fx.SetGroupLimit(ctx, k.GroupId, groupLimit)) + require.NoError(t, fx.SetSpaceLimit(ctx, k, spaceLimit)) + + groupInfo, err := fx.GroupInfo(ctx, k.GroupId) + require.NoError(t, err) + assert.Equal(t, groupLimit-spaceLimit, groupInfo.Limit) + assert.Equal(t, groupLimit, groupInfo.AccountLimit) + + spaceInfo, err := fx.SpaceInfo(ctx, k) + require.NoError(t, err) + assert.Equal(t, spaceLimit, spaceInfo.Limit) + }) + t.Run("isolate non-empty space", func(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + + k := newRandKey() + groupLimit := uint64(3000) + spaceLimit := uint64(1000) + require.NoError(t, fx.SetGroupLimit(ctx, k.GroupId, groupLimit)) + + bs := testutil.NewRandBlocks(3) + var sumSize uint64 + for _, b := range bs { + sumSize += uint64(len(b.RawData())) + } + k2 := Key{GroupId: k.GroupId, SpaceId: newRandKey().SpaceId} + + require.NoError(t, fx.BlocksAdd(ctx, bs)) + + // add first block to space 2 + fileId := testutil.NewRandCid().String() + cids, err := fx.CidEntriesByBlocks(ctx, bs[:1]) + require.NoError(t, err) + require.NoError(t, fx.FileBind(ctx, k2, fileId, cids)) + cids.Release() + + // add all blocks to space 1 + fileId = testutil.NewRandCid().String() + cids, err = fx.CidEntriesByBlocks(ctx, bs) + require.NoError(t, err) + require.NoError(t, fx.FileBind(ctx, k, fileId, cids)) + cids.Release() + + // isolate space + require.NoError(t, fx.SetSpaceLimit(ctx, k, spaceLimit)) + + groupInfo, err := fx.GroupInfo(ctx, k.GroupId) + require.NoError(t, err) + assert.Equal(t, groupLimit-spaceLimit, groupInfo.Limit) + assert.Equal(t, groupLimit, groupInfo.AccountLimit) + // only first common block consume group storage + assert.Equal(t, uint64(len(bs[0].RawData())), groupInfo.BytesUsage) + assert.Equal(t, uint64(1), groupInfo.CidsCount) + + spaceInfo, err := fx.SpaceInfo(ctx, k) + require.NoError(t, err) + assert.Equal(t, spaceLimit, spaceInfo.Limit) + assert.Equal(t, sumSize, spaceInfo.BytesUsage) + }) + t.Run("unite non-empty space", func(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + + k := newRandKey() + groupLimit := uint64(3000) + spaceLimit := uint64(1000) + require.NoError(t, fx.SetGroupLimit(ctx, k.GroupId, groupLimit)) + + bs := testutil.NewRandBlocks(3) + var sumSize uint64 + for _, b := range bs { + sumSize += uint64(len(b.RawData())) + } + k2 := Key{GroupId: k.GroupId, SpaceId: newRandKey().SpaceId} + + require.NoError(t, fx.BlocksAdd(ctx, bs)) + + // add first block to space 2 + fileId := testutil.NewRandCid().String() + cids, err := fx.CidEntriesByBlocks(ctx, bs[:1]) + require.NoError(t, err) + require.NoError(t, fx.FileBind(ctx, k2, fileId, cids)) + cids.Release() + + // add all blocks to space 1 + fileId = testutil.NewRandCid().String() + cids, err = fx.CidEntriesByBlocks(ctx, bs) + require.NoError(t, err) + require.NoError(t, fx.FileBind(ctx, k, fileId, cids)) + cids.Release() + + // isolate space + require.NoError(t, fx.SetSpaceLimit(ctx, k, spaceLimit)) + // unite space + require.NoError(t, fx.SetSpaceLimit(ctx, k, 0)) + + groupInfo, err := fx.GroupInfo(ctx, k.GroupId) + require.NoError(t, err) + assert.Equal(t, groupLimit, groupInfo.Limit) + assert.Equal(t, groupLimit, groupInfo.AccountLimit) + // only first common block consume group storage + assert.Equal(t, sumSize, groupInfo.BytesUsage) + assert.Equal(t, uint64(3), groupInfo.CidsCount) + + spaceInfo, err := fx.SpaceInfo(ctx, k) + require.NoError(t, err) + assert.Equal(t, uint64(0), spaceInfo.Limit) + assert.Equal(t, sumSize, spaceInfo.BytesUsage) + }) + t.Run("not enough space", func(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + + k := newRandKey() + groupLimit := uint64(3000) + require.NoError(t, fx.SetGroupLimit(ctx, k.GroupId, groupLimit)) + + bs := testutil.NewRandBlocks(3) + var sumSize uint64 + for _, b := range bs { + sumSize += uint64(len(b.RawData())) + } + require.NoError(t, fx.BlocksAdd(ctx, bs)) + + fileId := testutil.NewRandCid().String() + cids, err := fx.CidEntriesByBlocks(ctx, bs) + require.NoError(t, err) + require.NoError(t, fx.FileBind(ctx, k, fileId, cids)) + cids.Release() + + assert.EqualError(t, fx.SetSpaceLimit(ctx, k, sumSize-1), fileprotoerr.ErrNotEnoughSpace.Error()) + }) + t.Run("not enough space 2", func(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + + k := newRandKey() + groupLimit := uint64(1000) + require.NoError(t, fx.SetGroupLimit(ctx, k.GroupId, groupLimit)) + + assert.EqualError(t, fx.SetSpaceLimit(ctx, k, groupLimit+1), fileprotoerr.ErrNotEnoughSpace.Error()) + }) +} diff --git a/index/loader.go b/index/loader.go index b949b07..3f6562c 100644 --- a/index/loader.go +++ b/index/loader.go @@ -3,6 +3,7 @@ package index import ( "context" "errors" + "fmt" "math/rand" "strconv" "sync" @@ -74,8 +75,47 @@ func (ri *redisIndex) AcquireKey(ctx context.Context, key string) (exists bool, return } +func (ri *redisIndex) AcquireSpace(ctx context.Context, key Key) (entry groupSpaceEntry, release func(), err error) { + gExists, gRelease, err := ri.AcquireKey(ctx, groupKey(key)) + if err != nil { + return + } + sExists, sRelease, err := ri.AcquireKey(ctx, spaceKey(key)) + if err != nil { + gRelease() + return + } + + if entry.space, err = ri.getSpaceEntry(ctx, key); err != nil { + gRelease() + sRelease() + return + } + if entry.group, err = ri.getGroupEntry(ctx, key); err != nil { + gRelease() + sRelease() + return + } + + if entry.space.GetGroupId() != key.GroupId { + gRelease() + sRelease() + err = fmt.Errorf("space and group mismatched") + return + } + + release = func() { + gRelease() + sRelease() + } + entry.spaceExists = sExists + entry.groupExists = gExists + + return +} + func (ri *redisIndex) acquireKey(ctx context.Context, key string) (exists bool, release func(), err error) { - mu := ri.redsync.NewMutex("_lock:"+key, redsync.WithExpiry(time.Minute)) + mu := ri.redsync.NewMutex("_lock:"+key, redsync.WithExpiry(time.Minute*20)) if err = mu.LockContext(ctx); err != nil { return } diff --git a/index/loader_test.go b/index/loader_test.go index 1d1c512..9b11b4d 100644 --- a/index/loader_test.go +++ b/index/loader_test.go @@ -9,22 +9,21 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + "github.com/anyproto/any-sync-filenode/config" "github.com/anyproto/any-sync-filenode/testutil" ) func TestRedisIndex_PersistKeys(t *testing.T) { t.Run("no keys", func(t *testing.T) { - fx := newFixture(t) + fx := newFixtureConfig(t, &config.Config{PersistTtl: 2}) defer fx.Finish(t) - fx.persistTtl = time.Second * 2 bs := testutil.NewRandBlocks(5) require.NoError(t, fx.BlocksAdd(ctx, bs)) fx.PersistKeys(ctx) }) t.Run("persist", func(t *testing.T) { - fx := newFixture(t) + fx := newFixtureConfig(t, &config.Config{PersistTtl: 1}) defer fx.Finish(t) - fx.persistTtl = time.Second bs := testutil.NewRandBlocks(5) require.NoError(t, fx.BlocksAdd(ctx, bs)) for _, b := range bs { @@ -46,9 +45,9 @@ func TestRedisIndex_PersistKeys(t *testing.T) { } func TestRedisIndex_AcquireKey(t *testing.T) { - fx := newFixture(t) + fx := newFixtureConfig(t, &config.Config{PersistTtl: 1}) defer fx.Finish(t) - fx.persistTtl = time.Second + bs := testutil.NewRandBlocks(5) require.NoError(t, fx.BlocksAdd(ctx, bs)) for _, b := range bs { diff --git a/index/migrate.go b/index/migrate.go index 2362345..9a04c95 100644 --- a/index/migrate.go +++ b/index/migrate.go @@ -14,6 +14,7 @@ import ( "github.com/anyproto/any-sync-filenode/index/indexproto" ) +// Migrate from the version without groups func (ri *redisIndex) Migrate(ctx context.Context, key Key) (err error) { st := time.Now() // fast check before lock the key diff --git a/index/mock_index/mock_index.go b/index/mock_index/mock_index.go index 8b6b895..3c6a0fe 100644 --- a/index/mock_index/mock_index.go +++ b/index/mock_index/mock_index.go @@ -5,6 +5,7 @@ // // mockgen -destination mock_index/mock_index.go github.com/anyproto/any-sync-filenode/index Index // + // Package mock_index is a generated GoMock package. package mock_index @@ -86,6 +87,20 @@ func (mr *MockIndexMockRecorder) BlocksLock(arg0, arg1 any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BlocksLock", reflect.TypeOf((*MockIndex)(nil).BlocksLock), arg0, arg1) } +// CheckLimits mocks base method. +func (m *MockIndex) CheckLimits(arg0 context.Context, arg1 index.Key) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CheckLimits", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// CheckLimits indicates an expected call of CheckLimits. +func (mr *MockIndexMockRecorder) CheckLimits(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckLimits", reflect.TypeOf((*MockIndex)(nil).CheckLimits), arg0, arg1) +} + // CidEntries mocks base method. func (m *MockIndex) CidEntries(arg0 context.Context, arg1 []cid.Cid) (*index.CidEntries, error) { m.ctrl.T.Helper() @@ -213,6 +228,21 @@ func (mr *MockIndexMockRecorder) FileUnbind(arg0, arg1 any, arg2 ...any) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FileUnbind", reflect.TypeOf((*MockIndex)(nil).FileUnbind), varargs...) } +// FilesList mocks base method. +func (m *MockIndex) FilesList(arg0 context.Context, arg1 index.Key) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FilesList", arg0, arg1) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FilesList indicates an expected call of FilesList. +func (mr *MockIndexMockRecorder) FilesList(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FilesList", reflect.TypeOf((*MockIndex)(nil).FilesList), arg0, arg1) +} + // GroupInfo mocks base method. func (m *MockIndex) GroupInfo(arg0 context.Context, arg1 string) (index.GroupInfo, error) { m.ctrl.T.Helper() @@ -270,6 +300,23 @@ func (mr *MockIndexMockRecorder) Name() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockIndex)(nil).Name)) } +// OnBlockUploaded mocks base method. +func (m *MockIndex) OnBlockUploaded(arg0 context.Context, arg1 ...blocks.Block) { + m.ctrl.T.Helper() + varargs := []any{arg0} + for _, a := range arg1 { + varargs = append(varargs, a) + } + m.ctrl.Call(m, "OnBlockUploaded", varargs...) +} + +// OnBlockUploaded indicates an expected call of OnBlockUploaded. +func (mr *MockIndexMockRecorder) OnBlockUploaded(arg0 any, arg1 ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{arg0}, arg1...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OnBlockUploaded", reflect.TypeOf((*MockIndex)(nil).OnBlockUploaded), varargs...) +} + // Run mocks base method. func (m *MockIndex) Run(arg0 context.Context) error { m.ctrl.T.Helper() @@ -284,6 +331,34 @@ func (mr *MockIndexMockRecorder) Run(arg0 any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockIndex)(nil).Run), arg0) } +// SetGroupLimit mocks base method. +func (m *MockIndex) SetGroupLimit(arg0 context.Context, arg1 string, arg2 uint64) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetGroupLimit", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetGroupLimit indicates an expected call of SetGroupLimit. +func (mr *MockIndexMockRecorder) SetGroupLimit(arg0, arg1, arg2 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetGroupLimit", reflect.TypeOf((*MockIndex)(nil).SetGroupLimit), arg0, arg1, arg2) +} + +// SetSpaceLimit mocks base method. +func (m *MockIndex) SetSpaceLimit(arg0 context.Context, arg1 index.Key, arg2 uint64) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetSpaceLimit", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetSpaceLimit indicates an expected call of SetSpaceLimit. +func (mr *MockIndexMockRecorder) SetSpaceLimit(arg0, arg1, arg2 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetSpaceLimit", reflect.TypeOf((*MockIndex)(nil).SetSpaceLimit), arg0, arg1, arg2) +} + // SpaceDelete mocks base method. func (m *MockIndex) SpaceDelete(arg0 context.Context, arg1 index.Key) (bool, error) { m.ctrl.T.Helper() @@ -313,3 +388,17 @@ func (mr *MockIndexMockRecorder) SpaceInfo(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SpaceInfo", reflect.TypeOf((*MockIndex)(nil).SpaceInfo), arg0, arg1) } + +// WaitCidExists mocks base method. +func (m *MockIndex) WaitCidExists(arg0 context.Context, arg1 cid.Cid) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WaitCidExists", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// WaitCidExists indicates an expected call of WaitCidExists. +func (mr *MockIndexMockRecorder) WaitCidExists(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitCidExists", reflect.TypeOf((*MockIndex)(nil).WaitCidExists), arg0, arg1) +} diff --git a/index/pubsub.go b/index/pubsub.go new file mode 100644 index 0000000..14660b3 --- /dev/null +++ b/index/pubsub.go @@ -0,0 +1,90 @@ +package index + +import ( + "context" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/redis/go-redis/v9" + "go.uber.org/zap" +) + +const cidsChannel = "cidsChan" + +func (ri *redisIndex) WaitCidExists(ctx context.Context, k cid.Cid) (err error) { + ck := cidKey(k) + exists, release, err := ri.acquireKey(ctx, ck) + if err != nil { + return err + } + if exists { + release() + return nil + } + + ri.cidSubscriptionsMu.Lock() + release() + var ch = make(chan struct{}) + m := ri.cidSubscriptions[ck] + if m == nil { + m = make(map[chan struct{}]struct{}) + } + m[ch] = struct{}{} + ri.cidSubscriptions[ck] = m + ri.cidSubscriptionsMu.Unlock() + + cleanup := func() { + ri.cidSubscriptionsMu.Lock() + m := ri.cidSubscriptions[ck] + if m != nil { + delete(m, ch) + if len(m) == 0 { + delete(ri.cidSubscriptions, ck) + } + } + ri.cidSubscriptionsMu.Unlock() + } + + select { + case <-ctx.Done(): + cleanup() + return ctx.Err() + case <-ch: + return + } +} + +func (ri *redisIndex) OnBlockUploaded(ctx context.Context, bs ...blocks.Block) { + if _, err := ri.cl.Pipelined(ctx, func(pipe redis.Pipeliner) error { + for _, b := range bs { + _ = pipe.Publish(ctx, cidsChannel, cidKey(b.Cid())) + } + return nil + }); err != nil { + log.WarnCtx(ctx, "can't publish uploaded cids", zap.Error(err)) + } +} + +func (ri *redisIndex) subscription(ctx context.Context) { + sub := ri.cl.Subscribe(ctx, cidsChannel) + defer func() { + _ = sub.Unsubscribe(ctx, cidsChannel) + }() + ch := sub.Channel() + for msg := range ch { + ri.handleSubscriptionMessage(msg.Payload) + } +} + +func (ri *redisIndex) handleSubscriptionMessage(msg string) { + ri.cidSubscriptionsMu.Lock() + defer ri.cidSubscriptionsMu.Unlock() + + chans := ri.cidSubscriptions[msg] + if chans != nil { + for ch := range chans { + close(ch) + } + delete(ri.cidSubscriptions, msg) + } +} diff --git a/index/pubsub_test.go b/index/pubsub_test.go new file mode 100644 index 0000000..f7b15c0 --- /dev/null +++ b/index/pubsub_test.go @@ -0,0 +1,69 @@ +package index + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/any-sync-filenode/testutil" +) + +func TestRedisIndex_WaitCidExists(t *testing.T) { + t.Run("cid exists", func(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + + k := newRandKey() + bs := testutil.NewRandBlocks(1) + require.NoError(t, fx.BlocksAdd(ctx, bs)) + fileId := testutil.NewRandCid().String() + cids, err := fx.CidEntriesByBlocks(ctx, bs) + require.NoError(t, err) + require.NoError(t, fx.FileBind(ctx, k, fileId, cids)) + cids.Release() + + tCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + require.NoError(t, fx.WaitCidExists(tCtx, bs[0].Cid())) + }) + + t.Run("cid wait", func(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + bs := testutil.NewRandBlocks(2) + + wg := &sync.WaitGroup{} + wg.Add(3) + tCtx, cancel := context.WithTimeout(ctx, time.Second*5) + defer cancel() + go func() { + defer wg.Done() + require.NoError(t, fx.WaitCidExists(tCtx, bs[0].Cid())) + }() + go func() { + defer wg.Done() + require.NoError(t, fx.WaitCidExists(tCtx, bs[0].Cid())) + }() + go func() { + defer wg.Done() + require.NoError(t, fx.WaitCidExists(tCtx, bs[1].Cid())) + }() + time.Sleep(time.Second / 2) + fx.OnBlockUploaded(ctx, bs[0], bs[1]) + wg.Wait() + }) + + t.Run("ctx done", func(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + bs := testutil.NewRandBlocks(1) + tCtx, cancel := context.WithTimeout(ctx, time.Second/10) + defer cancel() + + assert.ErrorIs(t, fx.WaitCidExists(tCtx, bs[0].Cid()), context.DeadlineExceeded) + }) +} diff --git a/index/unbind.go b/index/unbind.go index 9e85751..4e258e5 100644 --- a/index/unbind.go +++ b/index/unbind.go @@ -8,25 +8,20 @@ import ( ) func (ri *redisIndex) FileUnbind(ctx context.Context, key Key, fileIds ...string) (err error) { - _, gRelease, err := ri.AcquireKey(ctx, groupKey(key)) + entry, release, err := ri.AcquireSpace(ctx, key) if err != nil { return } - defer gRelease() - _, sRelease, err := ri.AcquireKey(ctx, spaceKey(key)) - if err != nil { - return - } - defer sRelease() + defer release() for _, fileId := range fileIds { - if err = ri.fileUnbind(ctx, key, fileId); err != nil { + if err = ri.fileUnbind(ctx, key, entry, fileId); err != nil { return } } return } -func (ri *redisIndex) fileUnbind(ctx context.Context, key Key, fileId string) (err error) { +func (ri *redisIndex) fileUnbind(ctx context.Context, key Key, entry groupSpaceEntry, fileId string) (err error) { var ( sk = spaceKey(key) gk = groupKey(key) @@ -48,6 +43,8 @@ func (ri *redisIndex) fileUnbind(ctx context.Context, key Key, fileId string) (e } defer cids.Release() + isolatedSpace := entry.space.Limit != 0 + // fetch cid refs in one pipeline var ( groupCidRefs = make([]*redis.StringCmd, len(cids.entries)) @@ -55,7 +52,9 @@ func (ri *redisIndex) fileUnbind(ctx context.Context, key Key, fileId string) (e ) _, err = ri.cl.Pipelined(ctx, func(pipe redis.Pipeliner) error { for i, c := range cids.entries { - groupCidRefs[i] = pipe.HGet(ctx, gk, cidKey(c.Cid)) + if !isolatedSpace { + groupCidRefs[i] = pipe.HGet(ctx, gk, cidKey(c.Cid)) + } spaceCidRefs[i] = pipe.HGet(ctx, sk, cidKey(c.Cid)) } return nil @@ -64,16 +63,6 @@ func (ri *redisIndex) fileUnbind(ctx context.Context, key Key, fileId string) (e return } - // load group and space info - spaceInfo, err := ri.getSpaceEntry(ctx, key) - if err != nil { - return - } - groupInfo, err := ri.getGroupEntry(ctx, key) - if err != nil { - return - } - // update info and calculate changes var ( groupRemoveKeys = make([]string, 0, len(cids.entries)) @@ -83,29 +72,31 @@ func (ri *redisIndex) fileUnbind(ctx context.Context, key Key, fileId string) (e affectedCidIdx = make([]int, 0, len(cids.entries)) ) - spaceInfo.FileCount-- + entry.space.FileCount-- for i, c := range cids.entries { - res, err := groupCidRefs[i].Result() - if err != nil { - return err - } ck := cidKey(c.Cid) - if res == "1" { - groupRemoveKeys = append(groupRemoveKeys, ck) - groupInfo.Size_ -= c.Size_ - groupInfo.CidCount-- - affectedCidIdx = append(affectedCidIdx, i) - } else { - groupDecrKeys = append(groupDecrKeys, ck) + if !isolatedSpace { + res, err := groupCidRefs[i].Result() + if err != nil { + return err + } + if res == "1" { + groupRemoveKeys = append(groupRemoveKeys, ck) + entry.group.Size_ -= c.Size_ + entry.group.CidCount-- + } else { + groupDecrKeys = append(groupDecrKeys, ck) + } } - res, err = spaceCidRefs[i].Result() + res, err := spaceCidRefs[i].Result() if err != nil { return err } if res == "1" { spaceRemoveKeys = append(spaceRemoveKeys, ck) - spaceInfo.Size_ -= c.Size_ - spaceInfo.CidCount-- + entry.space.Size_ -= c.Size_ + entry.space.CidCount-- + affectedCidIdx = append(affectedCidIdx, i) } else { spaceDecrKeys = append(spaceDecrKeys, ck) } @@ -130,8 +121,8 @@ func (ri *redisIndex) fileUnbind(ctx context.Context, key Key, fileId string) (e tx.HIncrBy(ctx, gk, k, -1) } } - spaceInfo.Save(ctx, key, tx) - groupInfo.Save(ctx, key, tx) + entry.space.Save(ctx, key, tx) + entry.group.Save(ctx, tx) return nil }) diff --git a/limit/limit.go b/limit/limit.go deleted file mode 100644 index e05c134..0000000 --- a/limit/limit.go +++ /dev/null @@ -1,122 +0,0 @@ -//go:generate mockgen -destination mock_limit/mock_limit.go github.com/anyproto/any-sync-filenode/limit Limit -package limit - -import ( - "context" - "encoding/hex" - "fmt" - "github.com/anyproto/any-sync/app" - "github.com/anyproto/any-sync/app/ocache" - "github.com/anyproto/any-sync/coordinator/coordinatorclient" - "github.com/anyproto/any-sync/net/peer" - "go.uber.org/atomic" - "strings" - "time" -) - -const CName = "filenode.limit" - -func New() Limit { - return new(limit) -} - -type Limit interface { - Check(ctx context.Context, spaceId string) (limit uint64, storageKey string, err error) - app.ComponentRunnable -} - -type limit struct { - cl coordinatorclient.CoordinatorClient - cache ocache.OCache -} - -func (l *limit) Init(a *app.App) (err error) { - l.cl = a.MustComponent(coordinatorclient.CName).(coordinatorclient.CoordinatorClient) - l.cache = ocache.New(l.fetchLimit, - ocache.WithTTL(time.Minute*10), - ocache.WithGCPeriod(time.Minute*5), - ) - return nil -} - -func (l *limit) Name() (name string) { - return CName -} - -func (l *limit) Run(ctx context.Context) (err error) { - return -} - -func (l *limit) Check(ctx context.Context, spaceId string) (limit uint64, storageKey string, err error) { - identity, err := peer.CtxIdentity(ctx) - if err != nil { - return - } - obj, err := l.cache.Get(ctx, encodeId(spaceId, identity)) - if err != nil { - return - } - e := obj.(*entry) - return e.limit, e.storageKey, nil -} - -func (l *limit) fetchLimit(ctx context.Context, id string) (value ocache.Object, err error) { - spaceId, identity, err := decodeId(id) - if err != nil { - return - } - result, err := l.cl.FileLimitCheck(ctx, spaceId, identity) - if err != nil { - return nil, err - } - - if result.StorageKey == "" { - result.StorageKey = spaceId - } - return &entry{ - spaceId: spaceId, - identity: identity, - lastUsage: atomic.NewTime(time.Now()), - limit: result.Limit, - storageKey: result.StorageKey, - }, nil -} - -func (l *limit) Close(ctx context.Context) (err error) { - return l.cache.Close() -} - -func encodeId(spaceId string, identity []byte) string { - return spaceId + ":" + hex.EncodeToString(identity) -} - -func decodeId(id string) (spaceId string, identity []byte, err error) { - idx := strings.Index(id, ":") - if idx == -1 { - err = fmt.Errorf("unexpected limit id") - } - spaceId = id[:idx] - if identity, err = hex.DecodeString(id[idx+1:]); err != nil { - return - } - return -} - -type entry struct { - spaceId string - identity []byte - lastUsage *atomic.Time - limit uint64 - storageKey string -} - -func (e *entry) TryClose(objectTTL time.Duration) (res bool, err error) { - if time.Now().Sub(e.lastUsage.Load()) < objectTTL { - return false, nil - } - return true, e.Close() -} - -func (e *entry) Close() error { - return nil -} diff --git a/limit/limit_test.go b/limit/limit_test.go deleted file mode 100644 index fa91176..0000000 --- a/limit/limit_test.go +++ /dev/null @@ -1,71 +0,0 @@ -package limit - -import ( - "context" - "github.com/anyproto/any-sync/app" - "github.com/anyproto/any-sync/coordinator/coordinatorclient" - "github.com/anyproto/any-sync/coordinator/coordinatorclient/mock_coordinatorclient" - "github.com/anyproto/any-sync/coordinator/coordinatorproto" - "github.com/anyproto/any-sync/net/peer" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/mock/gomock" - "testing" -) - -var ctx = context.Background() - -func TestLimit_Check(t *testing.T) { - var spaceId = "122345.123" - var identity = []byte("identity") - t.Run("success space", func(t *testing.T) { - fx := newFixture(t) - defer fx.Finish(t) - fx.client.EXPECT().FileLimitCheck(gomock.Any(), spaceId, identity).Return(&coordinatorproto.FileLimitCheckResponse{ - Limit: 123, - StorageKey: "sk", - }, nil) - res, storageKey, err := fx.Check(peer.CtxWithIdentity(ctx, identity), spaceId) - require.NoError(t, err) - assert.Equal(t, uint64(123), res) - assert.Equal(t, "sk", storageKey) - res, storageKey, err = fx.Check(peer.CtxWithIdentity(ctx, identity), spaceId) - require.NoError(t, err) - assert.Equal(t, uint64(123), res) - assert.Equal(t, "sk", storageKey) - }) - t.Run("no identity", func(t *testing.T) { - fx := newFixture(t) - defer fx.Finish(t) - _, _, err := fx.Check(ctx, spaceId) - require.Error(t, err) - }) - -} - -func newFixture(t *testing.T) *fixture { - ctrl := gomock.NewController(t) - fx := &fixture{ - Limit: New(), - client: mock_coordinatorclient.NewMockCoordinatorClient(ctrl), - ctrl: ctrl, - a: new(app.App), - } - fx.client.EXPECT().Name().Return(coordinatorclient.CName).AnyTimes() - fx.client.EXPECT().Init(gomock.Any()).AnyTimes() - fx.a.Register(fx.client).Register(fx.Limit) - require.NoError(t, fx.a.Start(ctx)) - return fx -} - -type fixture struct { - Limit - client *mock_coordinatorclient.MockCoordinatorClient - ctrl *gomock.Controller - a *app.App -} - -func (fx *fixture) Finish(t *testing.T) { - require.NoError(t, fx.a.Close(ctx)) - fx.ctrl.Finish() -} diff --git a/limit/mock_limit/mock_limit.go b/limit/mock_limit/mock_limit.go deleted file mode 100644 index 982266a..0000000 --- a/limit/mock_limit/mock_limit.go +++ /dev/null @@ -1,112 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/anyproto/any-sync-filenode/limit (interfaces: Limit) -// -// Generated by this command: -// -// mockgen -destination mock_limit/mock_limit.go github.com/anyproto/any-sync-filenode/limit Limit -// -// Package mock_limit is a generated GoMock package. -package mock_limit - -import ( - context "context" - reflect "reflect" - - app "github.com/anyproto/any-sync/app" - gomock "go.uber.org/mock/gomock" -) - -// MockLimit is a mock of Limit interface. -type MockLimit struct { - ctrl *gomock.Controller - recorder *MockLimitMockRecorder -} - -// MockLimitMockRecorder is the mock recorder for MockLimit. -type MockLimitMockRecorder struct { - mock *MockLimit -} - -// NewMockLimit creates a new mock instance. -func NewMockLimit(ctrl *gomock.Controller) *MockLimit { - mock := &MockLimit{ctrl: ctrl} - mock.recorder = &MockLimitMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockLimit) EXPECT() *MockLimitMockRecorder { - return m.recorder -} - -// Check mocks base method. -func (m *MockLimit) Check(arg0 context.Context, arg1 string) (uint64, string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Check", arg0, arg1) - ret0, _ := ret[0].(uint64) - ret1, _ := ret[1].(string) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// Check indicates an expected call of Check. -func (mr *MockLimitMockRecorder) Check(arg0, arg1 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Check", reflect.TypeOf((*MockLimit)(nil).Check), arg0, arg1) -} - -// Close mocks base method. -func (m *MockLimit) Close(arg0 context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Close", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// Close indicates an expected call of Close. -func (mr *MockLimitMockRecorder) Close(arg0 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockLimit)(nil).Close), arg0) -} - -// Init mocks base method. -func (m *MockLimit) Init(arg0 *app.App) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Init", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// Init indicates an expected call of Init. -func (mr *MockLimitMockRecorder) Init(arg0 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockLimit)(nil).Init), arg0) -} - -// Name mocks base method. -func (m *MockLimit) Name() string { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Name") - ret0, _ := ret[0].(string) - return ret0 -} - -// Name indicates an expected call of Name. -func (mr *MockLimitMockRecorder) Name() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockLimit)(nil).Name)) -} - -// Run mocks base method. -func (m *MockLimit) Run(arg0 context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Run", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// Run indicates an expected call of Run. -func (mr *MockLimitMockRecorder) Run(arg0 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockLimit)(nil).Run), arg0) -} diff --git a/store/mock_store/mock_store.go b/store/mock_store/mock_store.go index 07c45c0..2f1a00b 100644 --- a/store/mock_store/mock_store.go +++ b/store/mock_store/mock_store.go @@ -5,6 +5,7 @@ // // mockgen -destination mock_store/mock_store.go github.com/anyproto/any-sync-filenode/store Store // + // Package mock_store is a generated GoMock package. package mock_store