From 65699395f9288be5b15334c9e8af87894d954b28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E8=B6=85=E8=B6=8A?= <993921@qq.com> Date: Sun, 19 May 2024 20:50:44 +0800 Subject: [PATCH] sync fix --- docs/docs.go | 62 ------------------- docs/swagger.json | 62 ------------------- docs/swagger.yaml | 39 ------------ .../community/node/{local_api.go => api.go} | 0 internal/community/node/delegate.go | 20 +++--- internal/community/node/remote_api.go | 39 ------------ internal/community/storage/member_itor.go | 8 +-- .../community/storage/member_itor_test.go | 9 ++- internal/community/storage/merge.go | 10 +++ .../web_server/controllers/account/v1/sync.go | 31 ---------- internal/web_server/routers/builder.go | 1 - 11 files changed, 30 insertions(+), 251 deletions(-) rename internal/community/node/{local_api.go => api.go} (100%) delete mode 100644 internal/community/node/remote_api.go create mode 100644 internal/community/storage/merge.go delete mode 100644 internal/web_server/controllers/account/v1/sync.go diff --git a/docs/docs.go b/docs/docs.go index 11b3dac..7d79013 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -53,45 +53,6 @@ const docTemplate = `{ } } }, - "/api/account/v1/sync": { - "get": { - "security": [ - { - "JWT": [] - } - ], - "description": "download accounts", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Account" - ], - "parameters": [ - { - "type": "integer", - "description": "how many accounts to download", - "name": "count", - "in": "query", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/storage.Member" - } - } - } - } - } - }, "/api/auth/v1/login": { "post": { "description": "Get AccessToken By ApiKey", @@ -164,29 +125,6 @@ const docTemplate = `{ "type": "string" } } - }, - "storage.Member": { - "type": "object", - "properties": { - "hashedAccount": { - "type": "string" - }, - "privateKeyVault": { - "type": "string" - }, - "publicKey": { - "type": "string" - }, - "rpcAddress": { - "type": "string" - }, - "rpcPort": { - "type": "integer" - }, - "version": { - "type": "integer" - } - } } }, "securityDefinitions": { diff --git a/docs/swagger.json b/docs/swagger.json index 92c6dd0..a9db45b 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -42,45 +42,6 @@ } } }, - "/api/account/v1/sync": { - "get": { - "security": [ - { - "JWT": [] - } - ], - "description": "download accounts", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Account" - ], - "parameters": [ - { - "type": "integer", - "description": "how many accounts to download", - "name": "count", - "in": "query", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/storage.Member" - } - } - } - } - } - }, "/api/auth/v1/login": { "post": { "description": "Get AccessToken By ApiKey", @@ -153,29 +114,6 @@ "type": "string" } } - }, - "storage.Member": { - "type": "object", - "properties": { - "hashedAccount": { - "type": "string" - }, - "privateKeyVault": { - "type": "string" - }, - "publicKey": { - "type": "string" - }, - "rpcAddress": { - "type": "string" - }, - "rpcPort": { - "type": "integer" - }, - "version": { - "type": "integer" - } - } } }, "securityDefinitions": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index e5d7057..f203797 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -11,21 +11,6 @@ definitions: apiKey: type: string type: object - storage.Member: - properties: - hashedAccount: - type: string - privateKeyVault: - type: string - publicKey: - type: string - rpcAddress: - type: string - rpcPort: - type: integer - version: - type: integer - type: object info: contact: name: AAStar Support @@ -52,30 +37,6 @@ paths: - JWT: [] tags: - Account - /api/account/v1/sync: - get: - consumes: - - application/json - description: download accounts - parameters: - - description: how many accounts to download - in: query - name: count - required: true - type: integer - produces: - - application/json - responses: - "200": - description: OK - schema: - items: - $ref: '#/definitions/storage.Member' - type: array - security: - - JWT: [] - tags: - - Account /api/auth/v1/login: post: consumes: diff --git a/internal/community/node/local_api.go b/internal/community/node/api.go similarity index 100% rename from internal/community/node/local_api.go rename to internal/community/node/api.go diff --git a/internal/community/node/delegate.go b/internal/community/node/delegate.go index 37d5217..876a65f 100644 --- a/internal/community/node/delegate.go +++ b/internal/community/node/delegate.go @@ -33,18 +33,18 @@ func (d *CommunityDelegate) GetBroadcasts(overhead, limit int) [][]byte { // LocalState return the local state data while a remote node joins or sync func (d *CommunityDelegate) LocalState(join bool) []byte { - if ss, err := storage.GetSnapshot(); err == nil { - if join { - return ss - } else { + skip := uint32(0) + if !join { + if ss, err := storage.GetSnapshot(); err == nil { if s, err := storage.UnmarshalSnapshot(ss); err == nil { - members := storage.GetAllMembers(s.TotalMembers) - if members != nil { - return storage.MarshalMembers(members) - } + skip = s.TotalMembers } } } + members := storage.GetMembers(skip, ^uint32(0)) + if len(members) > 0 { + return storage.MarshalMembers(members) + } return nil } @@ -52,8 +52,8 @@ func (d *CommunityDelegate) LocalState(join bool) []byte { func (d *CommunityDelegate) MergeRemoteState(buf []byte, join bool) { if len(buf) > 0 { if join { - if snap, err := storage.UnmarshalSnapshot(buf); err == nil { - go MergeRemoteMembers(snap) + if members := storage.UnmarshalMembers(buf); len(members) > 0 { + go storage.MergeRemoteAccounts(members) } } else { go func() { diff --git a/internal/community/node/remote_api.go b/internal/community/node/remote_api.go deleted file mode 100644 index 0667c98..0000000 --- a/internal/community/node/remote_api.go +++ /dev/null @@ -1,39 +0,0 @@ -package node - -import ( - "another_node/internal/community/storage" - "fmt" - "io" - "net/http" - "strings" -) - -const syncAccountApi = "api/account/v1/sync?count=" - -func splitNodeAddr(nodeAddr string) string { - u := strings.Split(nodeAddr, ":") - return u.Scheme + "://" + u.Host + ":8080" -} - -// MergeRemoteMembers merge remote members by calling the remote API /api/account/v{version}/sync -func MergeRemoteMembers(snap *storage.Snapshot) { - req, _ := http.NewRequest("GET", fmt.Sprintf("%s/%s%d", splitNodeAddr(entrypointNodeAddr[0]), syncAccountApi, snap.TotalMembers), nil) - - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - // handle error - return - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return - } - - if members := storage.UnmarshalMembers(body); members != nil { - for _, member := range members { - storage.MergeRemoteMember(&member) - } - } -} diff --git a/internal/community/storage/member_itor.go b/internal/community/storage/member_itor.go index 2c6585b..60e615d 100644 --- a/internal/community/storage/member_itor.go +++ b/internal/community/storage/member_itor.go @@ -4,7 +4,7 @@ import ( "github.com/syndtr/goleveldb/leveldb/util" ) -func GetAllMembers(total uint32) []Member { +func GetMembers(skip, size uint32) []Member { if ins, err := Open(); err != nil { return nil } else { @@ -15,16 +15,16 @@ func GetAllMembers(total uint32) []Member { iter := db.NewIterator(&util.Range{ Start: []byte(MemberPrefix), }, nil) - i := 0 + i := uint32(0) for iter.Next() { - if i < int(total) { + if i >= skip && i < skip+size { if m, err := Unmarshal(iter.Value()); err != nil { return nil } else { members = append(members, *m) } - i++ } + i++ } iter.Release() err = iter.Error() diff --git a/internal/community/storage/member_itor_test.go b/internal/community/storage/member_itor_test.go index 51f81f5..56b2052 100644 --- a/internal/community/storage/member_itor_test.go +++ b/internal/community/storage/member_itor_test.go @@ -4,15 +4,18 @@ import ( "os" "reflect" "testing" + + "github.com/google/uuid" ) func TestGetAllMembers(t *testing.T) { dir := os.TempDir() - os.Setenv("storage", dir+"/testing.dat") + uuid := uuid.New().String() + os.Setenv("storage", dir+"/testing.dat/"+uuid) defer func() { os.Unsetenv("storage") - os.Remove(dir + "/testing.dat") + os.RemoveAll(dir + "/testing.dat") }() member1 := &Member{ @@ -34,7 +37,7 @@ func TestGetAllMembers(t *testing.T) { UpsertMember(member2.HashedAccount, member2.PublicKey, "", member2.RpcAddress, member2.RpcPort, &member2.Version) // Call the GetAllMembers function - members := GetAllMembers(2) + members := GetMembers(0, ^uint32(0)) // Check the returned members expectedMembers := []Member{*member1, *member2} diff --git a/internal/community/storage/merge.go b/internal/community/storage/merge.go new file mode 100644 index 0000000..c35c0ed --- /dev/null +++ b/internal/community/storage/merge.go @@ -0,0 +1,10 @@ +package storage + +func MergeRemoteAccounts(accounts []Member) error { + for _, account := range accounts { + if err := MergeRemoteMember(&account); err != nil { + return err + } + } + return nil +} diff --git a/internal/web_server/controllers/account/v1/sync.go b/internal/web_server/controllers/account/v1/sync.go deleted file mode 100644 index 42586d5..0000000 --- a/internal/web_server/controllers/account/v1/sync.go +++ /dev/null @@ -1,31 +0,0 @@ -package account_v1 - -import ( - "another_node/internal/community/storage" - "another_node/internal/web_server/pkg" - "another_node/internal/web_server/pkg/response" - - "github.com/gin-gonic/gin" -) - -// Sync download accounts -// @Tags Account -// @Description download accounts -// @Accept json -// @Produce json -// @Success 200 {array} byte -// @Router /api/account/v1/sync [GET] -// @Param count query int true "how many accounts to download" -// @Security JWT -func Sync(ctx *gin.Context) { - if count, err := pkg.GetUIntParamFromQueryOrPath("count", ctx, false); err != nil { - response.BadRequest(ctx, err) - return - } else { - members := storage.GetAllMembers(uint32(count)) - - b := storage.MarshalMembers(members) - //community.SyncAccounts(count) - response.Success(ctx, b) - } -} diff --git a/internal/web_server/routers/builder.go b/internal/web_server/routers/builder.go index d9a2594..b44dfcc 100644 --- a/internal/web_server/routers/builder.go +++ b/internal/web_server/routers/builder.go @@ -26,6 +26,5 @@ func buildRouters(router *gin.Engine) { { account := router.Group("/api/account") account.POST("/v1/bind", account_v1.Bind) - account.GET("/v1/sync", account_v1.Sync) } }