From 3717be76d6f038a1e107767eb3312d231eefff80 Mon Sep 17 00:00:00 2001 From: Vadimka Komissarov Date: Sun, 4 Aug 2024 14:23:09 +0000 Subject: [PATCH 01/21] initial commit for release parsing from legacy redis cache --- cmd/alice/flags.go | 72 ++++++++++++- go.mod | 7 +- go.sum | 16 ++- internal/anilibria/anilibria.go | 16 +++ internal/anilibria/randomizer.go | 171 +++++++++++++++++++++++++++++++ internal/service/service.go | 4 + 6 files changed, 278 insertions(+), 8 deletions(-) create mode 100644 internal/anilibria/anilibria.go create mode 100644 internal/anilibria/randomizer.go diff --git a/cmd/alice/flags.go b/cmd/alice/flags.go index 3294b59..092214d 100644 --- a/cmd/alice/flags.go +++ b/cmd/alice/flags.go @@ -37,6 +37,7 @@ func flagsInitialization() []cli.Flag { Category: "Syslog settings", Usage: "optional setting; more information in syslog RFC", Value: "", + Hidden: true, }, // fiber-server settings @@ -63,6 +64,7 @@ func flagsInitialization() []cli.Flag { if enabled, the application will need to be ran through a shell because prefork mode sets environment variables; EXPERIMENTAL! USE CAREFULLY!`, + Hidden: true, DisableDefaultText: true, }, &cli.DurationFlag{ @@ -103,28 +105,33 @@ func flagsInitialization() []cli.Flag { &cli.BoolFlag{ Name: "limiter-enable", Category: "Limiter settings", + Hidden: true, DisableDefaultText: true, }, &cli.BoolFlag{ Name: "limiter-use-bbolt", Category: "Limiter settings", Usage: "use bbolt key\value file database instead of memory database", + Hidden: true, DisableDefaultText: true, }, &cli.BoolFlag{ Name: "limiter-bbolt-reset", Category: "Limiter settings", Usage: "if bbolt used as storage, reset all limited IPs on startup", + Hidden: true, DisableDefaultText: true, }, &cli.IntFlag{ Name: "limiter-max-req", Category: "Limiter settings", + Hidden: true, Value: 200, }, &cli.DurationFlag{ Name: "limiter-records-duration", Category: "Limiter settings", + Hidden: true, Value: 5 * time.Minute, }, @@ -258,8 +265,68 @@ func flagsInitialization() []cli.Flag { // "https://download.maxmind.com/geoip/databases/GeoLite2-Country/download?suffix=tar.gz.sha256" }, &cli.BoolFlag{ - Name: "geoip-download-sha256-skip", - Category: "GeoIP", + Name: "geoip-download-sha256-skip", + Category: "GeoIP", + DisableDefaultText: true, + }, + + // anilibria release randomizer + &cli.BoolFlag{ + Name: "randomizer-enable", + Category: "Release randomizer", + Usage: `alice has its own function for randomizing releases; it optimizes random_release (apiv1) + and public/random.php (www site), ensuring the high performance of these methods; + if disabled, all reuqests will be cached in shared cache pool with another methods`, + }, + &cli.StringFlag{ + Name: "randomizer-redis-host", + Category: "Release randomizer", + Value: "127.0.0.1:6279", + }, + &cli.StringFlag{ + Name: "randomizer-redis-password", + Category: "Release randomizer", + Value: "", + }, + &cli.IntFlag{ + Name: "randomizer-redis-database", + Category: "Release randomizer", + Value: 0, + }, + &cli.StringFlag{ + Name: "randomizer-releaseskey", + Category: "Release randomizer", + Usage: "the feature of the legacy", + Value: "apiInfo", + }, + &cli.DurationFlag{ + Name: "randomizer-update-frequency", + Category: "Release randomizer", + Value: 3 * time.Minute, + }, + &cli.IntFlag{ + Name: "redis-client-maxretries", + Category: "Release randomizer", + Hidden: true, + Value: 3, + }, + &cli.DurationFlag{ + Name: "redis-client-dialtimeout", + Category: "Release randomizer", + Hidden: true, + Value: 5 * time.Second, + }, + &cli.DurationFlag{ + Name: "redis-client-readtimeout", + Category: "Release randomizer", + Hidden: true, + Value: 3 * time.Second, + }, + &cli.DurationFlag{ + Name: "redis-client-writetimeout", + Category: "Release randomizer", + Hidden: true, + Value: 3 * time.Second, }, // custom settings @@ -267,6 +334,7 @@ func flagsInitialization() []cli.Flag { Name: "anilibrix-cmpb-mode", Category: "Feature flags", Usage: "avoiding 'Cannot POST //public/api/index.php' errors with req rewrite", + Hidden: true, DisableDefaultText: true, }, } diff --git a/go.mod b/go.mod index 2061882..a78ef9a 100644 --- a/go.mod +++ b/go.mod @@ -10,15 +10,18 @@ require ( github.com/klauspost/compress v1.17.9 github.com/mailru/easyjson v0.7.7 github.com/oschwald/maxminddb-golang v1.13.1 + github.com/redis/go-redis/v9 v9.6.1 github.com/rs/zerolog v1.33.0 - github.com/urfave/cli/v2 v2.27.2 + github.com/urfave/cli/v2 v2.27.3 github.com/valyala/bytebufferpool v1.0.0 github.com/valyala/fasthttp v1.55.0 ) require ( github.com/andybalholm/brotli v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/gofiber/utils/v2 v2.0.0-beta.3 // indirect github.com/google/uuid v1.5.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -30,7 +33,7 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/tinylib/msgp v1.1.8 // indirect github.com/valyala/tcplisten v1.0.0 // indirect - github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect + github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect go.etcd.io/bbolt v1.3.9 // indirect golang.org/x/sys v0.21.0 // indirect ) diff --git a/go.sum b/go.sum index 90ad68c..e8df57f 100644 --- a/go.sum +++ b/go.sum @@ -2,10 +2,16 @@ github.com/allegro/bigcache/v3 v3.1.0 h1:H2Vp8VOvxcrB91o86fUSVJFqeuz8kpyyB02eH3b github.com/allegro/bigcache/v3 v3.1.0/go.mod h1:aPyh7jEvrog9zAwx5N7+JUQX5dZTSGpxF1LAR4dr35I= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo= github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ= @@ -37,6 +43,8 @@ github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4= +github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -47,16 +55,16 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= -github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI= -github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM= +github.com/urfave/cli/v2 v2.27.3 h1:/POWahRmdh7uztQ3CYnaDddk0Rm90PyOgIxgW2rr41M= +github.com/urfave/cli/v2 v2.27.3/go.mod h1:m4QzxcD2qpra4z7WhzEGn74WZLViBnMpb1ToCAKdGRQ= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8= github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= -github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw= -github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk= +github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= +github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= diff --git a/internal/anilibria/anilibria.go b/internal/anilibria/anilibria.go new file mode 100644 index 0000000..7c0358d --- /dev/null +++ b/internal/anilibria/anilibria.go @@ -0,0 +1,16 @@ +package anilibria + +type ( + Releases map[string]*Release + Release struct { + Id uint + Code string + BlockedInfo *ReleaseBlockedInfo `json:"blockedInfo"` + } + ReleaseBlockedInfo struct { + Blocked bool + Reason string + IsBlockedInGeo []string `json:"is_blocked_in_geo"` + IsBlockedByCopyrights bool `json:"is_blocked_by_copyrights"` + } +) diff --git a/internal/anilibria/randomizer.go b/internal/anilibria/randomizer.go new file mode 100644 index 0000000..e169a60 --- /dev/null +++ b/internal/anilibria/randomizer.go @@ -0,0 +1,171 @@ +package anilibria + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "sync" + "time" + + "github.com/anilibria/alice/internal/utils" + futils "github.com/gofiber/fiber/v2/utils" + "github.com/redis/go-redis/v9" + "github.com/rs/zerolog" + "github.com/urfave/cli/v2" +) + +type Randomizer struct { + rctx context.Context + rclient *redis.Client + + releasesKey string + + log *zerolog.Logger + done func() <-chan struct{} + + mu sync.RWMutex + ready bool + releases []string +} + +func New(c context.Context) (_ *Randomizer) { + cli, log := + c.Value(utils.CKCliCtx).(*cli.Context), + c.Value(utils.CKLogger).(*zerolog.Logger) + + r := &Randomizer{} + + r.log, r.done = log, c.Done + + r.rctx = context.Background() + r.rclient = redis.NewClient(&redis.Options{ + Addr: cli.String("randomizer-redis-host"), + Password: cli.String("randomizer-redis-password"), + DB: cli.Int("randomizer-redis-database"), + + ClientName: fmt.Sprintf("%s/%s", cli.App.Name, cli.App.Version), + + MaxRetries: cli.Int("redis-client-maxretries"), + DialTimeout: cli.Duration("redis-client-dialtimeout"), + ReadTimeout: cli.Duration("redis-client-readtimeout"), + WriteTimeout: cli.Duration("redis-client-writetimeout"), + }) + + r.releasesKey = cli.String("randomizer-releaseskey") + r.releases = make([]string, 0) + + // + // + if _, e := r.lookupReleases(); e != nil { + r.log.Error().Msg(e.Error()) + r.done() + } + // + // + + return r +} + +func (m *Randomizer) Bootstrap() (e error) { + // add ping + // add timer + return m.destroy() +} + +func (*Randomizer) IsReady() bool { + return false +} + +// + +func (*Randomizer) loop() { + +} + +func (m *Randomizer) destroy() error { + return m.rclient.Close() +} + +func (m *Randomizer) peekReleaseKeyChunks() (_ int, e error) { + var res string + if res, e = m.rclient.Get(m.rctx, m.releasesKey).Result(); e == redis.Nil { + e = errors.New("no such release key in redis; is it correct - " + m.releasesKey) + return + } else if e != nil { + return + } else if res == "" { + e = errors.New("redis client respond with an empty string; is release key is alive?") + return + } + + return strconv.Atoi(res) +} + +func (m *Randomizer) lookupReleases() (_ []string, e error) { + var chunks int + if chunks, e = m.peekReleaseKeyChunks(); e != nil { + return + } else if chunks == 0 { + e = errors.New("invalid chunks count was responded by redis client or converted by golang") + return + } + m.log.Trace().Msgf("release key says about %d chunks", chunks) + + // avoid mass allocs + started := time.Now() + releases := make([]string, len(m.releases)) + res, errs, total, banned := "", []string{}, 0, 0 + + for i := 0; i < chunks; i++ { + m.log.Trace().Msgf("parsing chunk %d/%d...", i, chunks) + + if res, e = m.rclient.Get(m.rctx, m.releasesKey+strconv.Itoa(i)).Result(); e == redis.Nil { + e = errors.New(fmt.Sprintf("given chunk number %d is not exists", i)) + m.log.Warn().Msg(e.Error()) + errs = append(errs, e.Error()) + continue + } else if e != nil { + m.log.Warn().Msg("an error occured while peeking a releases chunk - " + e.Error()) + errs = append(errs, e.Error()) + continue + } + + var releasesChunk Releases + if e = json.Unmarshal(futils.UnsafeBytes(res), &releasesChunk); e != nil { + m.log.Warn().Msg("an error occured while unmarshal release chunk - " + e.Error()) + errs = append(errs, e.Error()) + continue + } + + for _, release := range releasesChunk { + if release.BlockedInfo != nil && release.BlockedInfo.IsBlockedByCopyrights { + m.log.Debug().Msgf("release %d (%s) worldwide banned, skip it...", release.Id, release.Code) + banned++ + continue + } + + if zerolog.GlobalLevel() <= zerolog.DebugLevel { + m.log.Trace().Msgf("release %d with code %s found", release.Id, release.Code) + } + + total++ + releases = append(releases, release.Code) + } + + } + + if errslen := len(errs); errslen != 0 { + m.log.Error().Msgf("%d chunks were corrupted, data from them did not get into the cache", errslen) + m.log.Error().Msg("release redis extraction process errors:") + + for _, err := range errs { + m.log.Error().Msg(err) + } + } + + m.log.Info().Msgf("in %s from %d (of %d) chunks added %d releases and %d skipped because of WW ban", + time.Since(started).String(), chunks-len(errs), chunks, total, banned) + return releases, nil +} diff --git a/internal/service/service.go b/internal/service/service.go index 3c3f3e1..f3dd70f 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -12,6 +12,7 @@ import ( "syscall" "time" + "github.com/anilibria/alice/internal/anilibria" "github.com/anilibria/alice/internal/cache" "github.com/anilibria/alice/internal/geoip" "github.com/anilibria/alice/internal/proxy" @@ -181,6 +182,9 @@ func (m *Service) Bootstrap() (e error) { gCtx = context.WithValue(gCtx, utils.CKCache, m.cache) gofunc(&wg, m.cache.Bootstrap) + // randomizer module + anilibria.New(gCtx) + // geoip module if gCli.Bool("geoip-enable") { if path := gCli.String("geoip-db-path"); path != "" { From 42ec56fa0efaf10c81a52b500aa2f74eaad76e44 Mon Sep 17 00:00:00 2001 From: Vadimka Komissarov Date: Fri, 9 Aug 2024 07:04:24 +0000 Subject: [PATCH 02/21] rendomizer flag enable use --- internal/service/service.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/service/service.go b/internal/service/service.go index f3dd70f..ff6f4bf 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -183,7 +183,9 @@ func (m *Service) Bootstrap() (e error) { gofunc(&wg, m.cache.Bootstrap) // randomizer module - anilibria.New(gCtx) + if gCli.Bool("randomizer-enable") { + anilibria.New(gCtx) + } // geoip module if gCli.Bool("geoip-enable") { @@ -233,6 +235,8 @@ func (m *Service) Bootstrap() (e error) { gLog.Info().Msg("ready...") + time.Sleep(time.Second) + wg.Wait() return m.loopError } From 9530c5b044d32bc0c7180dc081b9b18acb6af7ae Mon Sep 17 00:00:00 2001 From: Vadimka Komissarov Date: Fri, 9 Aug 2024 07:12:34 +0000 Subject: [PATCH 03/21] remove time.sleep --- internal/service/service.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/service/service.go b/internal/service/service.go index ff6f4bf..2ac47fd 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -235,8 +235,6 @@ func (m *Service) Bootstrap() (e error) { gLog.Info().Msg("ready...") - time.Sleep(time.Second) - wg.Wait() return m.loopError } From 9599ce0277aedfd92e2b8da91ad9100c873b4a75 Mon Sep 17 00:00:00 2001 From: Vadimka Komissarov Date: Sun, 11 Aug 2024 09:25:16 +0000 Subject: [PATCH 04/21] add randomize pages and methods for api --- internal/anilibria/randomizer.go | 114 ++++++++++++++++++++++++------- internal/proxy/handlers.go | 18 +++++ internal/proxy/middlewares.go | 12 +++- internal/proxy/proxy.go | 16 +++-- internal/proxy/validator.go | 14 +++- internal/service/router.go | 13 +++- internal/service/service.go | 14 ++-- internal/utils/context.go | 1 + 8 files changed, 166 insertions(+), 36 deletions(-) diff --git a/internal/anilibria/randomizer.go b/internal/anilibria/randomizer.go index e169a60..0ef7439 100644 --- a/internal/anilibria/randomizer.go +++ b/internal/anilibria/randomizer.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "math/rand" "strconv" "sync" "time" @@ -17,29 +18,28 @@ import ( ) type Randomizer struct { + log *zerolog.Logger + done func() <-chan struct{} + rctx context.Context rclient *redis.Client releasesKey string - - log *zerolog.Logger - done func() <-chan struct{} + relUpdFreq time.Duration mu sync.RWMutex ready bool releases []string } -func New(c context.Context) (_ *Randomizer) { +func New(c context.Context) (_ *Randomizer, e error) { cli, log := c.Value(utils.CKCliCtx).(*cli.Context), c.Value(utils.CKLogger).(*zerolog.Logger) r := &Randomizer{} - r.log, r.done = log, c.Done - r.rctx = context.Background() r.rclient = redis.NewClient(&redis.Options{ Addr: cli.String("randomizer-redis-host"), Password: cli.String("randomizer-redis-password"), @@ -53,39 +53,75 @@ func New(c context.Context) (_ *Randomizer) { WriteTimeout: cli.Duration("redis-client-writetimeout"), }) - r.releasesKey = cli.String("randomizer-releaseskey") - r.releases = make([]string, 0) + r.rctx = context.Background() + r.releases, r.releasesKey = make([]string, 0), cli.String("randomizer-releaseskey") - // - // - if _, e := r.lookupReleases(); e != nil { + if r.releases, e = r.lookupReleases(); e != nil { r.log.Error().Msg(e.Error()) - r.done() + return } - // - // + r.setReady(true) + + return r, e +} - return r +func (m *Randomizer) Bootstrap() { + m.loop() + m.destroy() } -func (m *Randomizer) Bootstrap() (e error) { - // add ping - // add timer - return m.destroy() +func (m *Randomizer) IsReady() bool { + return m.isReady() } -func (*Randomizer) IsReady() bool { - return false +func (m *Randomizer) Randomize() string { + return m.randomRelease() } // -func (*Randomizer) loop() { +func (m *Randomizer) loop() { + m.log.Debug().Msg("initiate randomizer release update loop...") + defer m.log.Debug().Msg("randomizer release update loop has been closed") + + update := time.NewTimer(m.relUpdFreq) + +LOOP: + for { + select { + case <-m.done(): + m.log.Info().Msg("internal abort() has been caught; initiate application closing...") + break LOOP + case <-update.C: + releases, e := m.lookupReleases() + if e != nil { + m.log.Error().Msg("could not updated releases for randomizer - " + e.Error()) + continue + } + + m.rotateReleases(releases) + } + } +} + +func (m *Randomizer) destroy() { + if e := m.rclient.Close(); e != nil { + m.log.Error().Msg("coudl not properly close http client - " + e.Error()) + } +} +func (m *Randomizer) setReady(ready bool) { + m.mu.Lock() + defer m.mu.Unlock() + + m.ready = ready } -func (m *Randomizer) destroy() error { - return m.rclient.Close() +func (m *Randomizer) isReady() bool { + m.mu.RLock() + defer m.mu.RUnlock() + + return m.ready } func (m *Randomizer) peekReleaseKeyChunks() (_ int, e error) { @@ -169,3 +205,33 @@ func (m *Randomizer) lookupReleases() (_ []string, e error) { time.Since(started).String(), chunks-len(errs), chunks, total, banned) return releases, nil } + +func (m *Randomizer) rotateReleases(releases []string) { + m.setReady(false) + m.mu.Lock() + + defer m.setReady(true) + defer m.mu.Unlock() + + m.log.Debug().Msgf("update current %d releases with slice of %d releases", + len(m.releases), len(releases)) + m.releases = releases +} + +func (m *Randomizer) randomRelease() (_ string) { + if !m.isReady() { + m.log.Warn().Msg("randomizer is not ready yet") + return + } + + if !m.mu.TryRLock() { + m.log.Warn().Msg("could not get randomized release, read lock is not available") + return + } + + m.mu.RLock() + defer m.mu.RUnlock() + + r := rand.Intn(len(m.releases)) + return m.releases[r] +} diff --git a/internal/proxy/handlers.go b/internal/proxy/handlers.go index b0db7e0..16a8810 100644 --- a/internal/proxy/handlers.go +++ b/internal/proxy/handlers.go @@ -35,6 +35,24 @@ func (m *Proxy) HandleProxyToDst(c *fiber.Ctx) (e error) { return } +func (m *Proxy) HandleRandomRelease(c *fiber.Ctx) (e error) { + if m.randomizer == nil { + return fiber.NewError(fiber.StatusServiceUnavailable, "BUG! randomizer is not initialized") + } + + if !m.randomizer.IsReady() { + return fiber.NewError(fiber.StatusServiceUnavailable, "randomizer is not ready yet") + } + + var release string + if release := m.randomizer.Randomize(); release == "" { + return fiber.NewError(fiber.StatusServiceUnavailable, "an error occured in randomizer") + } + + fmt.Fprintln(c, release) + return respondPlainWithStatus(c, fiber.StatusOK) +} + // internal api handlers func respondPlainWithStatus(c *fiber.Ctx, status int) error { diff --git a/internal/proxy/middlewares.go b/internal/proxy/middlewares.go index c2fc427..c1fd965 100644 --- a/internal/proxy/middlewares.go +++ b/internal/proxy/middlewares.go @@ -2,11 +2,12 @@ package proxy import ( "bytes" + "fmt" "github.com/gofiber/fiber/v2" ) -func (*Proxy) MiddlewareValidation(c *fiber.Ctx) (e error) { +func (m *Proxy) MiddlewareValidation(c *fiber.Ctx) (e error) { v := AcquireValidator(c, c.Request().Header.ContentType()) defer ReleaseValidator(v) @@ -14,6 +15,15 @@ func (*Proxy) MiddlewareValidation(c *fiber.Ctx) (e error) { return fiber.NewError(fiber.StatusBadRequest, e.Error()) } + if v.IsQueryEqual([]byte("random_release")) { + if m.randomizer != nil && m.randomizer.IsReady() { + if release := m.randomizer.Randomize(); release != "" { + fmt.Fprintln(c, release) + return respondPlainWithStatus(c, fiber.StatusOK) + } + } + } + // continue request processing e = c.Next() return diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index e20f95a..226a44a 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "github.com/anilibria/alice/internal/anilibria" "github.com/anilibria/alice/internal/cache" "github.com/anilibria/alice/internal/geoip" "github.com/anilibria/alice/internal/utils" @@ -19,8 +20,9 @@ type Proxy struct { client *ProxyClient config *ProxyConfig - cache *cache.Cache - geoip geoip.GeoIPClient + cache *cache.Cache + geoip geoip.GeoIPClient + randomizer *anilibria.Randomizer } type ProxyConfig struct { @@ -31,6 +33,11 @@ type ProxyConfig struct { func NewProxy(c context.Context) *Proxy { cli := c.Value(utils.CKCliCtx).(*cli.Context) + var randomizer *anilibria.Randomizer + if c.Value(utils.CKRandomizer) != nil { + randomizer = c.Value(utils.CKRandomizer).(*anilibria.Randomizer) + } + return &Proxy{ client: NewClient(cli), config: &ProxyConfig{ @@ -39,8 +46,9 @@ func NewProxy(c context.Context) *Proxy { apiSecret: []byte(cli.String("cache-api-secret")), }, - cache: c.Value(utils.CKCache).(*cache.Cache), - geoip: c.Value(utils.CKGeoIP).(geoip.GeoIPClient), + cache: c.Value(utils.CKCache).(*cache.Cache), + geoip: c.Value(utils.CKGeoIP).(geoip.GeoIPClient), + randomizer: randomizer, } } diff --git a/internal/proxy/validator.go b/internal/proxy/validator.go index 2f5114b..487180a 100644 --- a/internal/proxy/validator.go +++ b/internal/proxy/validator.go @@ -82,6 +82,10 @@ func (m *Validator) ValidateRequest() (e error) { return } +func (m *Validator) IsQueryEqual(equal []byte) bool { + return m.isQueryEqual(equal) +} + func (m *Validator) Reset() { m.Context().RemoveUserValue(utils.UVCacheKey) ReleaseKey(m.cacheKey) @@ -116,7 +120,6 @@ func (m *Validator) validateContentType() utils.RequestContentType { } func (m *Validator) validateCustomHeaders() { - for header, ch := range Stoch { val := m.Request().Header.PeekBytes(futils.UnsafeBytes(header)) if len(val) != 0 { @@ -275,3 +278,12 @@ func (m *Validator) isQueryWhitelisted() (ok bool) { return } + +func (m *Validator) isQueryEqual(equal []byte) (_ bool) { + var query []byte + if query = m.requestArgs.PeekBytes([]byte("query")); len(query) == 0 { + return + } + + return bytes.Equal(query, equal) +} diff --git a/internal/service/router.go b/internal/service/router.go index 9103cc2..07224d3 100644 --- a/internal/service/router.go +++ b/internal/service/router.go @@ -172,15 +172,24 @@ func (m *Service) fiberRouterInitialization() { cacheapi.Post("/purge", m.proxy.HandleCachePurge) cacheapi.Post("/purgeall", m.proxy.HandleCachePurgeAll) + // + // ALICE randomizer method for legacy www + if m.randomizer != nil { + m.fb.Post("/public/random.php", m.proxy.HandleRandomRelease) + } + // // ALICE apiv1 requests proxying lifecycle: // step1 - validate request apiv1 := m.fb.Group("/public/api", m.proxy.MiddlewareValidation) - // step2 - check cache availability and try to respond with it + // step2 - reroute random() queries: + // + + // step3 - check cache availability and try to respond with it apiv1.Use(skip.New(m.proxy.HandleProxyToCache, m.proxy.IsRequestCached)) - // step3 - proxy request to upstream + // step4 - proxy request to upstream apiv1.Use(m.proxy.HandleProxyToDst) } diff --git a/internal/service/service.go b/internal/service/service.go index 2ac47fd..f721d9e 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -38,9 +38,10 @@ type Service struct { fb *fiber.App fbstor fiber.Storage - proxy *proxy.Proxy - cache *cache.Cache - geoip geoip.GeoIPClient + proxy *proxy.Proxy + cache *cache.Cache + geoip geoip.GeoIPClient + randomizer *anilibria.Randomizer syslogWriter io.Writer @@ -184,7 +185,12 @@ func (m *Service) Bootstrap() (e error) { // randomizer module if gCli.Bool("randomizer-enable") { - anilibria.New(gCtx) + if m.randomizer, e = anilibria.New(gCtx); e != nil { + return + } + + gCtx = context.WithValue(gCtx, utils.CKRandomizer, m.randomizer) + gofunc(&wg, m.randomizer.Bootstrap) } // geoip module diff --git a/internal/utils/context.go b/internal/utils/context.go index 8df8d38..99b78c7 100644 --- a/internal/utils/context.go +++ b/internal/utils/context.go @@ -8,4 +8,5 @@ const ( CKAbortFunc CKCache CKGeoIP + CKRandomizer ) From 6d005306dc25dcec9118ae3f98b03bd2399bc5b7 Mon Sep 17 00:00:00 2001 From: Vadimka Komissarov Date: Fri, 9 Aug 2024 07:53:17 +0000 Subject: [PATCH 05/21] fix panic while geoip is disabled --- internal/proxy/proxy.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 226a44a..8042371 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -38,6 +38,11 @@ func NewProxy(c context.Context) *Proxy { randomizer = c.Value(utils.CKRandomizer).(*anilibria.Randomizer) } + var gip geoip.GeoIPClient + if c.Value(utils.CKGeoIP) != nil { + gip = c.Value(utils.CKGeoIP).(geoip.GeoIPClient) + } + return &Proxy{ client: NewClient(cli), config: &ProxyConfig{ @@ -46,9 +51,10 @@ func NewProxy(c context.Context) *Proxy { apiSecret: []byte(cli.String("cache-api-secret")), }, - cache: c.Value(utils.CKCache).(*cache.Cache), - geoip: c.Value(utils.CKGeoIP).(geoip.GeoIPClient), + geoip: gip, randomizer: randomizer, + + cache: c.Value(utils.CKCache).(*cache.Cache), } } From fa2ad9b877e7207008937fce05ddc3a88246935e Mon Sep 17 00:00:00 2001 From: Vadimka Komissarov Date: Sun, 11 Aug 2024 09:57:19 +0000 Subject: [PATCH 06/21] refactoring, move initial parsing in bootstrap, bugfixes --- cmd/alice/flags.go | 8 ++- internal/anilibria/randomizer.go | 83 ++++++++++++++++++++------------ internal/service/service.go | 6 +-- 3 files changed, 62 insertions(+), 35 deletions(-) diff --git a/cmd/alice/flags.go b/cmd/alice/flags.go index 092214d..2b57406 100644 --- a/cmd/alice/flags.go +++ b/cmd/alice/flags.go @@ -302,7 +302,13 @@ func flagsInitialization() []cli.Flag { &cli.DurationFlag{ Name: "randomizer-update-frequency", Category: "Release randomizer", - Value: 3 * time.Minute, + Value: 1 * time.Minute, + }, + &cli.DurationFlag{ + Name: "randomizer-update-frequency-onerror", + Category: "Release randomizer", + Value: 1 * time.Minute, + Hidden: true, }, &cli.IntFlag{ Name: "redis-client-maxretries", diff --git a/internal/anilibria/randomizer.go b/internal/anilibria/randomizer.go index 0ef7439..82962cf 100644 --- a/internal/anilibria/randomizer.go +++ b/internal/anilibria/randomizer.go @@ -18,51 +18,52 @@ import ( ) type Randomizer struct { - log *zerolog.Logger - done func() <-chan struct{} + log *zerolog.Logger + done func() <-chan struct{} + abort context.CancelFunc rctx context.Context rclient *redis.Client - releasesKey string - relUpdFreq time.Duration + releasesKey string + relUpdFreq time.Duration + relUpdFreqErr time.Duration mu sync.RWMutex ready bool releases []string } -func New(c context.Context) (_ *Randomizer, e error) { - cli, log := - c.Value(utils.CKCliCtx).(*cli.Context), - c.Value(utils.CKLogger).(*zerolog.Logger) +func New(c context.Context) *Randomizer { + cli := c.Value(utils.CKCliCtx).(*cli.Context) - r := &Randomizer{} - r.log, r.done = log, c.Done + r := &Randomizer{ + done: c.Done, + log: c.Value(utils.CKLogger).(*zerolog.Logger), + abort: c.Value(utils.CKAbortFunc).(context.CancelFunc), - r.rclient = redis.NewClient(&redis.Options{ - Addr: cli.String("randomizer-redis-host"), - Password: cli.String("randomizer-redis-password"), - DB: cli.Int("randomizer-redis-database"), + rctx: context.Background(), + rclient: redis.NewClient(&redis.Options{ + Addr: cli.String("randomizer-redis-host"), + Password: cli.String("randomizer-redis-password"), + DB: cli.Int("randomizer-redis-database"), - ClientName: fmt.Sprintf("%s/%s", cli.App.Name, cli.App.Version), + ClientName: fmt.Sprintf("%s/%s", cli.App.Name, cli.App.Version), - MaxRetries: cli.Int("redis-client-maxretries"), - DialTimeout: cli.Duration("redis-client-dialtimeout"), - ReadTimeout: cli.Duration("redis-client-readtimeout"), - WriteTimeout: cli.Duration("redis-client-writetimeout"), - }) + MaxRetries: cli.Int("redis-client-maxretries"), + DialTimeout: cli.Duration("redis-client-dialtimeout"), + ReadTimeout: cli.Duration("redis-client-readtimeout"), + WriteTimeout: cli.Duration("redis-client-writetimeout"), + }), - r.rctx = context.Background() - r.releases, r.releasesKey = make([]string, 0), cli.String("randomizer-releaseskey") + relUpdFreq: cli.Duration("randomizer-update-frequency"), + relUpdFreqErr: cli.Duration("randomizer-update-frequency-onerror"), - if r.releases, e = r.lookupReleases(); e != nil { - r.log.Error().Msg(e.Error()) - return + releases: make([]string, 0), + releasesKey: cli.String("randomizer-releaseskey"), } - r.setReady(true) - return r, e + return r } func (m *Randomizer) Bootstrap() { @@ -84,6 +85,15 @@ func (m *Randomizer) loop() { m.log.Debug().Msg("initiate randomizer release update loop...") defer m.log.Debug().Msg("randomizer release update loop has been closed") + // initial parsing + var e error + if m.releases, e = m.lookupReleases(); e != nil { + m.log.Error().Msg(e.Error()) + m.abort() + return + } + m.setReady(true) + update := time.NewTimer(m.relUpdFreq) LOOP: @@ -93,13 +103,17 @@ LOOP: m.log.Info().Msg("internal abort() has been caught; initiate application closing...") break LOOP case <-update.C: - releases, e := m.lookupReleases() - if e != nil { + update.Stop() + + var releases []string + if releases, e = m.lookupReleases(); e != nil { m.log.Error().Msg("could not updated releases for randomizer - " + e.Error()) + update.Reset(m.relUpdFreqErr) continue } m.rotateReleases(releases) + update.Reset(m.relUpdFreq) } } } @@ -148,15 +162,24 @@ func (m *Randomizer) lookupReleases() (_ []string, e error) { return } m.log.Trace().Msgf("release key says about %d chunks", chunks) + m.log.Info().Msgf("release parsing from redis with %d chunks started", chunks) // avoid mass allocs started := time.Now() - releases := make([]string, len(m.releases)) + releases := make([]string, 0, len(m.releases)) res, errs, total, banned := "", []string{}, 0, 0 for i := 0; i < chunks; i++ { m.log.Trace().Msgf("parsing chunk %d/%d...", i, chunks) + select { + case <-m.done(): + e = errors.New("chunk parsing has been interrupted by global abort()") + return + default: + break + } + if res, e = m.rclient.Get(m.rctx, m.releasesKey+strconv.Itoa(i)).Result(); e == redis.Nil { e = errors.New(fmt.Sprintf("given chunk number %d is not exists", i)) m.log.Warn().Msg(e.Error()) diff --git a/internal/service/service.go b/internal/service/service.go index f721d9e..e9cefbd 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -185,11 +185,9 @@ func (m *Service) Bootstrap() (e error) { // randomizer module if gCli.Bool("randomizer-enable") { - if m.randomizer, e = anilibria.New(gCtx); e != nil { - return - } - + m.randomizer = anilibria.New(gCtx) gCtx = context.WithValue(gCtx, utils.CKRandomizer, m.randomizer) + gofunc(&wg, m.randomizer.Bootstrap) } From 07b0939aeb82be453e98b67fda0fb1e0b18bb59f Mon Sep 17 00:00:00 2001 From: Vadimka Komissarov Date: Sun, 11 Aug 2024 10:42:57 +0000 Subject: [PATCH 07/21] add apiv1 reponse format for apiv1 methods --- internal/proxy/middlewares.go | 12 ++- internal/proxy/validator.go | 3 +- internal/utils/apiv1_response.go | 25 ++++- internal/utils/apiv1_response_easyjson.go | 110 +++++++++++++++++----- 4 files changed, 124 insertions(+), 26 deletions(-) diff --git a/internal/proxy/middlewares.go b/internal/proxy/middlewares.go index c1fd965..4fb004d 100644 --- a/internal/proxy/middlewares.go +++ b/internal/proxy/middlewares.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" + "github.com/anilibria/alice/internal/utils" "github.com/gofiber/fiber/v2" ) @@ -16,14 +17,23 @@ func (m *Proxy) MiddlewareValidation(c *fiber.Ctx) (e error) { } if v.IsQueryEqual([]byte("random_release")) { + fmt.Println("this is random_release") if m.randomizer != nil && m.randomizer.IsReady() { if release := m.randomizer.Randomize(); release != "" { fmt.Fprintln(c, release) - return respondPlainWithStatus(c, fiber.StatusOK) + fmt.Println("all good - ", release) + if e = utils.RespondWithRandomRelease(fiber.StatusOK, release, c); e == nil { + return respondPlainWithStatus(c, fiber.StatusOK) + } + rlog(c).Error().Msg("could not respond on random release query - " + e.Error()) } + fmt.Println("randomizer empty result") } + fmt.Println("randomizer falls") } + // {"status":true,"data":{"code":"nande-koko-ni-sensei-ga"},"error":null} + // continue request processing e = c.Next() return diff --git a/internal/proxy/validator.go b/internal/proxy/validator.go index 487180a..74d95be 100644 --- a/internal/proxy/validator.go +++ b/internal/proxy/validator.go @@ -23,6 +23,7 @@ type Validator struct { requestArgs *fasthttp.Args + query []byte cacheKey *Key customs CustomHeaders @@ -43,6 +44,7 @@ func AcquireValidator(c *fiber.Ctx, ctr []byte) (v *Validator) { } func ReleaseValidator(v *Validator) { + fasthttp.ReleaseArgs(v.requestArgs) v.Reset() validatorPool.Put(v) } @@ -60,7 +62,6 @@ func (m *Validator) ValidateRequest() (e error) { m.validateCustomHeaders() m.requestArgs = fasthttp.AcquireArgs() - defer fasthttp.ReleaseArgs(m.requestArgs) if e = m.extractRequestKey(); e != nil { return diff --git a/internal/utils/apiv1_response.go b/internal/utils/apiv1_response.go index 10c31f0..da97a94 100644 --- a/internal/utils/apiv1_response.go +++ b/internal/utils/apiv1_response.go @@ -10,7 +10,7 @@ import ( type ( ApiResponse struct { Status bool - Data interface{} + Data *ApiResponseData Error *ApiError } ApiResponseWOData struct { @@ -18,6 +18,9 @@ type ( Data interface{} `json:"-"` Error *ApiError } + ApiResponseData struct { + Code string + } ApiError struct { Code int Message string @@ -46,6 +49,7 @@ func ReleaseApiResponseWOData(ar *ApiResponseWOData) { var apiResponsePool = sync.Pool{ New: func() interface{} { return &ApiResponse{ + Data: &ApiResponseData{}, Error: &ApiError{}, } }, @@ -57,7 +61,9 @@ func AcquireApiResponse() *ApiResponse { func ReleaseApiResponse(ar *ApiResponse) { ar.Status = false - ar.Error.Code, ar.Error.Message, ar.Error.Description = 0, "", "" + if ar.Error != nil { + ar.Error.Code, ar.Error.Message, ar.Error.Description = 0, "", "" + } apiResponsePool.Put(ar) } @@ -76,6 +82,21 @@ func RespondWithApiError(status int, msg, desc string, w io.Writer) (e error) { return } +func RespondWithRandomRelease(status int, code string, w io.Writer) (e error) { + apirsp := AcquireApiResponse() + defer ReleaseApiResponse(apirsp) + + apirsp.Status, apirsp.Data.Code, apirsp.Error = true, code, nil + + var buf []byte + if buf, e = easyjson.Marshal(apirsp); e != nil { + return + } + + _, e = w.Write(buf) + return +} + func UnmarshalApiResponse(payload []byte) (_ *ApiResponseWOData, e error) { apirsp := AcquireApiResponseWOData() return apirsp, easyjson.Unmarshal(payload, apirsp) diff --git a/internal/utils/apiv1_response_easyjson.go b/internal/utils/apiv1_response_easyjson.go index 182f259..99f0793 100644 --- a/internal/utils/apiv1_response_easyjson.go +++ b/internal/utils/apiv1_response_easyjson.go @@ -102,7 +102,73 @@ func (v *ApiResponseWOData) UnmarshalJSON(data []byte) error { func (v *ApiResponseWOData) UnmarshalEasyJSON(l *jlexer.Lexer) { easyjson1e840bfDecodeGithubComAnilibriaAliceInternalUtils(l, v) } -func easyjson1e840bfDecodeGithubComAnilibriaAliceInternalUtils1(in *jlexer.Lexer, out *ApiResponse) { +func easyjson1e840bfDecodeGithubComAnilibriaAliceInternalUtils1(in *jlexer.Lexer, out *ApiResponseData) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "code": + out.Code = string(in.String()) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjson1e840bfEncodeGithubComAnilibriaAliceInternalUtils1(out *jwriter.Writer, in ApiResponseData) { + out.RawByte('{') + first := true + _ = first + { + const prefix string = ",\"code\":" + out.RawString(prefix[1:]) + out.String(string(in.Code)) + } + out.RawByte('}') +} + +// MarshalJSON supports json.Marshaler interface +func (v ApiResponseData) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + easyjson1e840bfEncodeGithubComAnilibriaAliceInternalUtils1(&w, v) + return w.Buffer.BuildBytes(), w.Error +} + +// MarshalEasyJSON supports easyjson.Marshaler interface +func (v ApiResponseData) MarshalEasyJSON(w *jwriter.Writer) { + easyjson1e840bfEncodeGithubComAnilibriaAliceInternalUtils1(w, v) +} + +// UnmarshalJSON supports json.Unmarshaler interface +func (v *ApiResponseData) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + easyjson1e840bfDecodeGithubComAnilibriaAliceInternalUtils1(&r, v) + return r.Error() +} + +// UnmarshalEasyJSON supports easyjson.Unmarshaler interface +func (v *ApiResponseData) UnmarshalEasyJSON(l *jlexer.Lexer) { + easyjson1e840bfDecodeGithubComAnilibriaAliceInternalUtils1(l, v) +} +func easyjson1e840bfDecodeGithubComAnilibriaAliceInternalUtils2(in *jlexer.Lexer, out *ApiResponse) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -124,12 +190,14 @@ func easyjson1e840bfDecodeGithubComAnilibriaAliceInternalUtils1(in *jlexer.Lexer case "status": out.Status = bool(in.Bool()) case "data": - if m, ok := out.Data.(easyjson.Unmarshaler); ok { - m.UnmarshalEasyJSON(in) - } else if m, ok := out.Data.(json.Unmarshaler); ok { - _ = m.UnmarshalJSON(in.Raw()) + if in.IsNull() { + in.Skip() + out.Data = nil } else { - out.Data = in.Interface() + if out.Data == nil { + out.Data = new(ApiResponseData) + } + (*out.Data).UnmarshalEasyJSON(in) } case "error": if in.IsNull() { @@ -151,7 +219,7 @@ func easyjson1e840bfDecodeGithubComAnilibriaAliceInternalUtils1(in *jlexer.Lexer in.Consumed() } } -func easyjson1e840bfEncodeGithubComAnilibriaAliceInternalUtils1(out *jwriter.Writer, in ApiResponse) { +func easyjson1e840bfEncodeGithubComAnilibriaAliceInternalUtils2(out *jwriter.Writer, in ApiResponse) { out.RawByte('{') first := true _ = first @@ -163,12 +231,10 @@ func easyjson1e840bfEncodeGithubComAnilibriaAliceInternalUtils1(out *jwriter.Wri { const prefix string = ",\"data\":" out.RawString(prefix) - if m, ok := in.Data.(easyjson.Marshaler); ok { - m.MarshalEasyJSON(out) - } else if m, ok := in.Data.(json.Marshaler); ok { - out.Raw(m.MarshalJSON()) + if in.Data == nil { + out.RawString("null") } else { - out.Raw(json.Marshal(in.Data)) + (*in.Data).MarshalEasyJSON(out) } } { @@ -186,27 +252,27 @@ func easyjson1e840bfEncodeGithubComAnilibriaAliceInternalUtils1(out *jwriter.Wri // MarshalJSON supports json.Marshaler interface func (v ApiResponse) MarshalJSON() ([]byte, error) { w := jwriter.Writer{} - easyjson1e840bfEncodeGithubComAnilibriaAliceInternalUtils1(&w, v) + easyjson1e840bfEncodeGithubComAnilibriaAliceInternalUtils2(&w, v) return w.Buffer.BuildBytes(), w.Error } // MarshalEasyJSON supports easyjson.Marshaler interface func (v ApiResponse) MarshalEasyJSON(w *jwriter.Writer) { - easyjson1e840bfEncodeGithubComAnilibriaAliceInternalUtils1(w, v) + easyjson1e840bfEncodeGithubComAnilibriaAliceInternalUtils2(w, v) } // UnmarshalJSON supports json.Unmarshaler interface func (v *ApiResponse) UnmarshalJSON(data []byte) error { r := jlexer.Lexer{Data: data} - easyjson1e840bfDecodeGithubComAnilibriaAliceInternalUtils1(&r, v) + easyjson1e840bfDecodeGithubComAnilibriaAliceInternalUtils2(&r, v) return r.Error() } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *ApiResponse) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjson1e840bfDecodeGithubComAnilibriaAliceInternalUtils1(l, v) + easyjson1e840bfDecodeGithubComAnilibriaAliceInternalUtils2(l, v) } -func easyjson1e840bfDecodeGithubComAnilibriaAliceInternalUtils2(in *jlexer.Lexer, out *ApiError) { +func easyjson1e840bfDecodeGithubComAnilibriaAliceInternalUtils3(in *jlexer.Lexer, out *ApiError) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -241,7 +307,7 @@ func easyjson1e840bfDecodeGithubComAnilibriaAliceInternalUtils2(in *jlexer.Lexer in.Consumed() } } -func easyjson1e840bfEncodeGithubComAnilibriaAliceInternalUtils2(out *jwriter.Writer, in ApiError) { +func easyjson1e840bfEncodeGithubComAnilibriaAliceInternalUtils3(out *jwriter.Writer, in ApiError) { out.RawByte('{') first := true _ = first @@ -266,23 +332,23 @@ func easyjson1e840bfEncodeGithubComAnilibriaAliceInternalUtils2(out *jwriter.Wri // MarshalJSON supports json.Marshaler interface func (v ApiError) MarshalJSON() ([]byte, error) { w := jwriter.Writer{} - easyjson1e840bfEncodeGithubComAnilibriaAliceInternalUtils2(&w, v) + easyjson1e840bfEncodeGithubComAnilibriaAliceInternalUtils3(&w, v) return w.Buffer.BuildBytes(), w.Error } // MarshalEasyJSON supports easyjson.Marshaler interface func (v ApiError) MarshalEasyJSON(w *jwriter.Writer) { - easyjson1e840bfEncodeGithubComAnilibriaAliceInternalUtils2(w, v) + easyjson1e840bfEncodeGithubComAnilibriaAliceInternalUtils3(w, v) } // UnmarshalJSON supports json.Unmarshaler interface func (v *ApiError) UnmarshalJSON(data []byte) error { r := jlexer.Lexer{Data: data} - easyjson1e840bfDecodeGithubComAnilibriaAliceInternalUtils2(&r, v) + easyjson1e840bfDecodeGithubComAnilibriaAliceInternalUtils3(&r, v) return r.Error() } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *ApiError) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjson1e840bfDecodeGithubComAnilibriaAliceInternalUtils2(l, v) + easyjson1e840bfDecodeGithubComAnilibriaAliceInternalUtils3(l, v) } From 652be9e13fbcd2f06417f542b644fdc120ace28e Mon Sep 17 00:00:00 2001 From: Vadimka Komissarov Date: Sun, 11 Aug 2024 12:55:29 +0000 Subject: [PATCH 08/21] dead lock fix --- internal/anilibria/randomizer.go | 42 ++++++++++++++------------------ internal/proxy/handlers.go | 7 ++---- internal/proxy/middlewares.go | 2 +- 3 files changed, 21 insertions(+), 30 deletions(-) diff --git a/internal/anilibria/randomizer.go b/internal/anilibria/randomizer.go index 82962cf..3034b45 100644 --- a/internal/anilibria/randomizer.go +++ b/internal/anilibria/randomizer.go @@ -71,9 +71,9 @@ func (m *Randomizer) Bootstrap() { m.destroy() } -func (m *Randomizer) IsReady() bool { - return m.isReady() -} +// func (m *Randomizer) IsReady() bool { +// return m.isReady() +// } func (m *Randomizer) Randomize() string { return m.randomRelease() @@ -92,7 +92,6 @@ func (m *Randomizer) loop() { m.abort() return } - m.setReady(true) update := time.NewTimer(m.relUpdFreq) @@ -124,19 +123,19 @@ func (m *Randomizer) destroy() { } } -func (m *Randomizer) setReady(ready bool) { - m.mu.Lock() - defer m.mu.Unlock() +// func (m *Randomizer) setReady(ready bool) { +// m.mu.Lock() +// defer m.mu.Unlock() - m.ready = ready -} +// m.ready = ready +// } -func (m *Randomizer) isReady() bool { - m.mu.RLock() - defer m.mu.RUnlock() +// func (m *Randomizer) isReady() bool { +// m.mu.RLock() +// defer m.mu.RUnlock() - return m.ready -} +// return m.ready +// } func (m *Randomizer) peekReleaseKeyChunks() (_ int, e error) { var res string @@ -230,10 +229,7 @@ func (m *Randomizer) lookupReleases() (_ []string, e error) { } func (m *Randomizer) rotateReleases(releases []string) { - m.setReady(false) m.mu.Lock() - - defer m.setReady(true) defer m.mu.Unlock() m.log.Debug().Msgf("update current %d releases with slice of %d releases", @@ -242,19 +238,17 @@ func (m *Randomizer) rotateReleases(releases []string) { } func (m *Randomizer) randomRelease() (_ string) { - if !m.isReady() { - m.log.Warn().Msg("randomizer is not ready yet") - return - } - if !m.mu.TryRLock() { m.log.Warn().Msg("could not get randomized release, read lock is not available") return } - - m.mu.RLock() defer m.mu.RUnlock() + if len(m.releases) == 0 { + m.log.Warn().Msg("randomizer is not ready yet") + return + } + r := rand.Intn(len(m.releases)) return m.releases[r] } diff --git a/internal/proxy/handlers.go b/internal/proxy/handlers.go index 16a8810..4000416 100644 --- a/internal/proxy/handlers.go +++ b/internal/proxy/handlers.go @@ -40,13 +40,10 @@ func (m *Proxy) HandleRandomRelease(c *fiber.Ctx) (e error) { return fiber.NewError(fiber.StatusServiceUnavailable, "BUG! randomizer is not initialized") } - if !m.randomizer.IsReady() { - return fiber.NewError(fiber.StatusServiceUnavailable, "randomizer is not ready yet") - } - var release string if release := m.randomizer.Randomize(); release == "" { - return fiber.NewError(fiber.StatusServiceUnavailable, "an error occured in randomizer") + return fiber.NewError(fiber.StatusServiceUnavailable, + "an error occured in randomizer, maybe it's not ready yet") } fmt.Fprintln(c, release) diff --git a/internal/proxy/middlewares.go b/internal/proxy/middlewares.go index 4fb004d..28ab4d7 100644 --- a/internal/proxy/middlewares.go +++ b/internal/proxy/middlewares.go @@ -18,7 +18,7 @@ func (m *Proxy) MiddlewareValidation(c *fiber.Ctx) (e error) { if v.IsQueryEqual([]byte("random_release")) { fmt.Println("this is random_release") - if m.randomizer != nil && m.randomizer.IsReady() { + if m.randomizer != nil { if release := m.randomizer.Randomize(); release != "" { fmt.Fprintln(c, release) fmt.Println("all good - ", release) From 0fb8eadd7c052c2d39bdef4bd4442efaa97d2477 Mon Sep 17 00:00:00 2001 From: Vadimka Komissarov Date: Sun, 11 Aug 2024 14:41:03 +0000 Subject: [PATCH 09/21] datarace fix --- internal/anilibria/randomizer.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/anilibria/randomizer.go b/internal/anilibria/randomizer.go index 3034b45..f17fffe 100644 --- a/internal/anilibria/randomizer.go +++ b/internal/anilibria/randomizer.go @@ -86,12 +86,13 @@ func (m *Randomizer) loop() { defer m.log.Debug().Msg("randomizer release update loop has been closed") // initial parsing - var e error - if m.releases, e = m.lookupReleases(); e != nil { + releases, e := m.lookupReleases() + if e != nil { m.log.Error().Msg(e.Error()) m.abort() return } + m.rotateReleases(releases) update := time.NewTimer(m.relUpdFreq) From a15635234899e39ad98838d79da445522e542fc5 Mon Sep 17 00:00:00 2001 From: Vadimka Komissarov Date: Sun, 11 Aug 2024 14:41:45 +0000 Subject: [PATCH 10/21] debug remove --- internal/proxy/middlewares.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/internal/proxy/middlewares.go b/internal/proxy/middlewares.go index 28ab4d7..414f29d 100644 --- a/internal/proxy/middlewares.go +++ b/internal/proxy/middlewares.go @@ -2,7 +2,6 @@ package proxy import ( "bytes" - "fmt" "github.com/anilibria/alice/internal/utils" "github.com/gofiber/fiber/v2" @@ -17,19 +16,14 @@ func (m *Proxy) MiddlewareValidation(c *fiber.Ctx) (e error) { } if v.IsQueryEqual([]byte("random_release")) { - fmt.Println("this is random_release") if m.randomizer != nil { if release := m.randomizer.Randomize(); release != "" { - fmt.Fprintln(c, release) - fmt.Println("all good - ", release) if e = utils.RespondWithRandomRelease(fiber.StatusOK, release, c); e == nil { return respondPlainWithStatus(c, fiber.StatusOK) } rlog(c).Error().Msg("could not respond on random release query - " + e.Error()) } - fmt.Println("randomizer empty result") } - fmt.Println("randomizer falls") } // {"status":true,"data":{"code":"nande-koko-ni-sensei-ga"},"error":null} From 2bd8c6d3b8019fc829440c258dff9309d213547b Mon Sep 17 00:00:00 2001 From: Vadimka Komissarov Date: Sun, 11 Aug 2024 14:41:57 +0000 Subject: [PATCH 11/21] comment remove --- internal/proxy/middlewares.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/proxy/middlewares.go b/internal/proxy/middlewares.go index 414f29d..f6efd61 100644 --- a/internal/proxy/middlewares.go +++ b/internal/proxy/middlewares.go @@ -26,8 +26,6 @@ func (m *Proxy) MiddlewareValidation(c *fiber.Ctx) (e error) { } } - // {"status":true,"data":{"code":"nande-koko-ni-sensei-ga"},"error":null} - // continue request processing e = c.Next() return From 3d7d99f1e960eb6eabe267957e05467b1fb4adbb Mon Sep 17 00:00:00 2001 From: Vadimka Komissarov Date: Sun, 11 Aug 2024 15:00:03 +0000 Subject: [PATCH 12/21] small after test fixes --- internal/anilibria/randomizer.go | 4 ++-- internal/proxy/handlers.go | 2 +- internal/utils/apiv1_response.go | 7 ++++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/internal/anilibria/randomizer.go b/internal/anilibria/randomizer.go index f17fffe..58faadf 100644 --- a/internal/anilibria/randomizer.go +++ b/internal/anilibria/randomizer.go @@ -120,7 +120,7 @@ LOOP: func (m *Randomizer) destroy() { if e := m.rclient.Close(); e != nil { - m.log.Error().Msg("coudl not properly close http client - " + e.Error()) + m.log.Error().Msg("could not properly close http client - " + e.Error()) } } @@ -162,7 +162,7 @@ func (m *Randomizer) lookupReleases() (_ []string, e error) { return } m.log.Trace().Msgf("release key says about %d chunks", chunks) - m.log.Info().Msgf("release parsing from redis with %d chunks started", chunks) + m.log.Info().Msgf("staring release parsing from redis with %d chunks", chunks) // avoid mass allocs started := time.Now() diff --git a/internal/proxy/handlers.go b/internal/proxy/handlers.go index 4000416..92f18fc 100644 --- a/internal/proxy/handlers.go +++ b/internal/proxy/handlers.go @@ -41,7 +41,7 @@ func (m *Proxy) HandleRandomRelease(c *fiber.Ctx) (e error) { } var release string - if release := m.randomizer.Randomize(); release == "" { + if release = m.randomizer.Randomize(); release == "" { return fiber.NewError(fiber.StatusServiceUnavailable, "an error occured in randomizer, maybe it's not ready yet") } diff --git a/internal/utils/apiv1_response.go b/internal/utils/apiv1_response.go index da97a94..7619aaa 100644 --- a/internal/utils/apiv1_response.go +++ b/internal/utils/apiv1_response.go @@ -64,14 +64,19 @@ func ReleaseApiResponse(ar *ApiResponse) { if ar.Error != nil { ar.Error.Code, ar.Error.Message, ar.Error.Description = 0, "", "" } + if ar.Data != nil { + ar.Data.Code = "" + } apiResponsePool.Put(ar) } func RespondWithApiError(status int, msg, desc string, w io.Writer) (e error) { apirsp := AcquireApiResponse() + defer ReleaseApiResponse(apirsp) + apirsp.Error.Code, apirsp.Error.Message, apirsp.Error.Description = status, msg, desc - defer ReleaseApiResponse(apirsp) + apirsp.Data = nil var buf []byte if buf, e = easyjson.Marshal(apirsp); e != nil { From 20fced1a4162b2e558a8b38858bf50cd31625c17 Mon Sep 17 00:00:00 2001 From: Vadimka Komissarov Date: Sun, 11 Aug 2024 15:04:01 +0000 Subject: [PATCH 13/21] fix SCC-SA4011 --- internal/anilibria/randomizer.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/internal/anilibria/randomizer.go b/internal/anilibria/randomizer.go index 58faadf..642aea1 100644 --- a/internal/anilibria/randomizer.go +++ b/internal/anilibria/randomizer.go @@ -170,14 +170,12 @@ func (m *Randomizer) lookupReleases() (_ []string, e error) { res, errs, total, banned := "", []string{}, 0, 0 for i := 0; i < chunks; i++ { - m.log.Trace().Msgf("parsing chunk %d/%d...", i, chunks) - select { case <-m.done(): e = errors.New("chunk parsing has been interrupted by global abort()") return default: - break + m.log.Trace().Msgf("parsing chunk %d/%d...", i, chunks) } if res, e = m.rclient.Get(m.rctx, m.releasesKey+strconv.Itoa(i)).Result(); e == redis.Nil { From 3ee8ddc160e90a254ca3751986d261cf60de3235 Mon Sep 17 00:00:00 2001 From: Vadimka Komissarov Date: Sun, 11 Aug 2024 15:04:34 +0000 Subject: [PATCH 14/21] fix SCC-S1028 --- internal/anilibria/randomizer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/anilibria/randomizer.go b/internal/anilibria/randomizer.go index 642aea1..7e14a6b 100644 --- a/internal/anilibria/randomizer.go +++ b/internal/anilibria/randomizer.go @@ -179,7 +179,7 @@ func (m *Randomizer) lookupReleases() (_ []string, e error) { } if res, e = m.rclient.Get(m.rctx, m.releasesKey+strconv.Itoa(i)).Result(); e == redis.Nil { - e = errors.New(fmt.Sprintf("given chunk number %d is not exists", i)) + e = fmt.Errorf("given chunk number %d is not exists", i) m.log.Warn().Msg(e.Error()) errs = append(errs, e.Error()) continue From 93ffec3db750212417705fa61b3a7ebae6f7d4e2 Mon Sep 17 00:00:00 2001 From: Vadimka Komissarov Date: Sun, 11 Aug 2024 15:05:48 +0000 Subject: [PATCH 15/21] fix GO-W1027 --- internal/anilibria/randomizer.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/anilibria/randomizer.go b/internal/anilibria/randomizer.go index 7e14a6b..7677005 100644 --- a/internal/anilibria/randomizer.go +++ b/internal/anilibria/randomizer.go @@ -167,7 +167,10 @@ func (m *Randomizer) lookupReleases() (_ []string, e error) { // avoid mass allocs started := time.Now() releases := make([]string, 0, len(m.releases)) - res, errs, total, banned := "", []string{}, 0, 0 + + var res string + var errs []string + var total, banned int for i := 0; i < chunks; i++ { select { From 9dc52c6c9f5038226864f04921744b826f9a3efe Mon Sep 17 00:00:00 2001 From: Vadimka Komissarov Date: Sun, 11 Aug 2024 15:08:33 +0000 Subject: [PATCH 16/21] fix RVV-B0001 --- internal/proxy/validator.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/proxy/validator.go b/internal/proxy/validator.go index 74d95be..1943d5e 100644 --- a/internal/proxy/validator.go +++ b/internal/proxy/validator.go @@ -84,7 +84,7 @@ func (m *Validator) ValidateRequest() (e error) { } func (m *Validator) IsQueryEqual(equal []byte) bool { - return m.isQueryEqual(equal) + return m.queryLookup(equal) } func (m *Validator) Reset() { @@ -280,7 +280,7 @@ func (m *Validator) isQueryWhitelisted() (ok bool) { return } -func (m *Validator) isQueryEqual(equal []byte) (_ bool) { +func (m *Validator) queryLookup(equal []byte) (_ bool) { var query []byte if query = m.requestArgs.PeekBytes([]byte("query")); len(query) == 0 { return From d7979cd9b3b552a677b3eca042e66a9f00eb51da Mon Sep 17 00:00:00 2001 From: Vadimka Komissarov Date: Sun, 11 Aug 2024 15:09:07 +0000 Subject: [PATCH 17/21] fix RVV-B0012 --- internal/proxy/middlewares.go | 2 +- internal/utils/apiv1_response.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/proxy/middlewares.go b/internal/proxy/middlewares.go index f6efd61..cf43466 100644 --- a/internal/proxy/middlewares.go +++ b/internal/proxy/middlewares.go @@ -18,7 +18,7 @@ func (m *Proxy) MiddlewareValidation(c *fiber.Ctx) (e error) { if v.IsQueryEqual([]byte("random_release")) { if m.randomizer != nil { if release := m.randomizer.Randomize(); release != "" { - if e = utils.RespondWithRandomRelease(fiber.StatusOK, release, c); e == nil { + if e = utils.RespondWithRandomRelease(release, c); e == nil { return respondPlainWithStatus(c, fiber.StatusOK) } rlog(c).Error().Msg("could not respond on random release query - " + e.Error()) diff --git a/internal/utils/apiv1_response.go b/internal/utils/apiv1_response.go index 7619aaa..e320627 100644 --- a/internal/utils/apiv1_response.go +++ b/internal/utils/apiv1_response.go @@ -87,7 +87,7 @@ func RespondWithApiError(status int, msg, desc string, w io.Writer) (e error) { return } -func RespondWithRandomRelease(status int, code string, w io.Writer) (e error) { +func RespondWithRandomRelease(code string, w io.Writer) (e error) { apirsp := AcquireApiResponse() defer ReleaseApiResponse(apirsp) From aab27791fad0e7e6369709b4b1e31962ed67ada5 Mon Sep 17 00:00:00 2001 From: Vadimka Komissarov Date: Sun, 11 Aug 2024 15:09:52 +0000 Subject: [PATCH 18/21] fix RVV-B0012 --- internal/proxy/validator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/proxy/validator.go b/internal/proxy/validator.go index 1943d5e..e57d21c 100644 --- a/internal/proxy/validator.go +++ b/internal/proxy/validator.go @@ -243,7 +243,7 @@ func (m *Validator) isArgsWhitelisted() (_ bool) { declinedKeysPtr := declinedKeysPool.Get().(*[]string) declinedKeys := *declinedKeysPtr - m.requestArgs.VisitAll(func(key, value []byte) { + m.requestArgs.VisitAll(func(key, _ []byte) { if _, ok := postArgsWhitelist[futils.UnsafeString(key)]; !ok { declinedKeys = append(declinedKeys, futils.UnsafeString(key)) } From 44f6e165c9e592fa5a42acd2c4f8b70b6031ace2 Mon Sep 17 00:00:00 2001 From: Vadimka Komissarov Date: Sun, 11 Aug 2024 15:12:24 +0000 Subject: [PATCH 19/21] skip GSC-G404 --- internal/anilibria/randomizer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/anilibria/randomizer.go b/internal/anilibria/randomizer.go index 7677005..7816f1b 100644 --- a/internal/anilibria/randomizer.go +++ b/internal/anilibria/randomizer.go @@ -251,6 +251,6 @@ func (m *Randomizer) randomRelease() (_ string) { return } - r := rand.Intn(len(m.releases)) + r := rand.Intn(len(m.releases)) // skipcq: GSC-G404 math/rand is enoght here return m.releases[r] } From a4e80f12018222ce191948999ecec028c411d1ba Mon Sep 17 00:00:00 2001 From: Vadimka Komissarov Date: Sun, 11 Aug 2024 15:12:52 +0000 Subject: [PATCH 20/21] fix SCC-U1000 --- internal/anilibria/randomizer.go | 1 - internal/proxy/validator.go | 1 - 2 files changed, 2 deletions(-) diff --git a/internal/anilibria/randomizer.go b/internal/anilibria/randomizer.go index 7816f1b..ec933e9 100644 --- a/internal/anilibria/randomizer.go +++ b/internal/anilibria/randomizer.go @@ -30,7 +30,6 @@ type Randomizer struct { relUpdFreqErr time.Duration mu sync.RWMutex - ready bool releases []string } diff --git a/internal/proxy/validator.go b/internal/proxy/validator.go index e57d21c..a9eff41 100644 --- a/internal/proxy/validator.go +++ b/internal/proxy/validator.go @@ -23,7 +23,6 @@ type Validator struct { requestArgs *fasthttp.Args - query []byte cacheKey *Key customs CustomHeaders From cc57f30d1789016d36a4a455038083712cb15c35 Mon Sep 17 00:00:00 2001 From: Vadimka Komissarov Date: Sun, 11 Aug 2024 15:17:23 +0000 Subject: [PATCH 21/21] after review comment remove --- internal/anilibria/randomizer.go | 18 ------------------ internal/service/router.go | 7 ++----- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/internal/anilibria/randomizer.go b/internal/anilibria/randomizer.go index ec933e9..c047e99 100644 --- a/internal/anilibria/randomizer.go +++ b/internal/anilibria/randomizer.go @@ -70,10 +70,6 @@ func (m *Randomizer) Bootstrap() { m.destroy() } -// func (m *Randomizer) IsReady() bool { -// return m.isReady() -// } - func (m *Randomizer) Randomize() string { return m.randomRelease() } @@ -123,20 +119,6 @@ func (m *Randomizer) destroy() { } } -// func (m *Randomizer) setReady(ready bool) { -// m.mu.Lock() -// defer m.mu.Unlock() - -// m.ready = ready -// } - -// func (m *Randomizer) isReady() bool { -// m.mu.RLock() -// defer m.mu.RUnlock() - -// return m.ready -// } - func (m *Randomizer) peekReleaseKeyChunks() (_ int, e error) { var res string if res, e = m.rclient.Get(m.rctx, m.releasesKey).Result(); e == redis.Nil { diff --git a/internal/service/router.go b/internal/service/router.go index 07224d3..7fd7893 100644 --- a/internal/service/router.go +++ b/internal/service/router.go @@ -184,12 +184,9 @@ func (m *Service) fiberRouterInitialization() { // step1 - validate request apiv1 := m.fb.Group("/public/api", m.proxy.MiddlewareValidation) - // step2 - reroute random() queries: - // - - // step3 - check cache availability and try to respond with it + // step2 - check cache availability and try to respond with it apiv1.Use(skip.New(m.proxy.HandleProxyToCache, m.proxy.IsRequestCached)) - // step4 - proxy request to upstream + // step3 - proxy request to upstream apiv1.Use(m.proxy.HandleProxyToDst) }