From 77c2f461f2e87436befd2667716e885a8499b1f6 Mon Sep 17 00:00:00 2001 From: slhmy Date: Sun, 14 Jan 2024 20:43:18 +0800 Subject: [PATCH] Remove asynq & perf makefile --- Makefile | 89 ++++----- go.mod | 3 - go.sum | 23 --- src/application/asynq_worker/main.go | 16 -- src/application/rpc_server/main.go | 2 + src/core/agent/asynq/client.go | 28 --- src/core/agent/asynq/middleware.go | 22 --- src/core/agent/asynq/priority.go | 16 -- src/core/agent/asynq/redis.go | 16 -- src/core/agent/asynq/scheduler.go | 31 --- src/core/agent/asynq/server.go | 40 ---- src/service/business/judger_task.go | 252 ------------------------ src/service/proto/oj_lab_greeting.proto | 2 + src/service/proto/oj_lab_stream.proto | 2 + 14 files changed, 47 insertions(+), 495 deletions(-) delete mode 100644 src/application/asynq_worker/main.go delete mode 100644 src/core/agent/asynq/client.go delete mode 100644 src/core/agent/asynq/middleware.go delete mode 100644 src/core/agent/asynq/priority.go delete mode 100644 src/core/agent/asynq/redis.go delete mode 100644 src/core/agent/asynq/scheduler.go delete mode 100644 src/core/agent/asynq/server.go delete mode 100644 src/service/business/judger_task.go diff --git a/Makefile b/Makefile index 57a957f..d2b9835 100644 --- a/Makefile +++ b/Makefile @@ -5,46 +5,25 @@ help: @echo "Usage: make [target]" @echo "" @echo "Targets:" - @echo " build - Build the application" - @echo " clear-db - Clear the database" - @echo " setup-db - Setup the database" + @echo " build - Build the application, swagger document will be generated" @echo " check - Run go vet" - @echo " test - Run tests" - -.PHONY: get-front -get-front: - ./scripts/update-frontend-dist.sh artifacts/oj-lab-front/dist - -.PHONY: install-tools -install-tools: - go install github.com/swaggo/swag/cmd/swag@latest - @# Referencing https://grpc.io/docs/protoc-installation/ - @./scripts/install-protoc.sh - @# Track https://grpc.io/docs/languages/go/quickstart/ for update - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 - -.PHONY: gen-proto -gen-proto: install-tools - protoc --go_out=. --go_opt=paths=source_relative \ - --go-grpc_out=. --go-grpc_opt=paths=source_relative \ - src/service/proto/*.proto - -.PHONY: gen-swagger -gen-swagger: install-tools - swag fmt -d src/application/server - swag init -d src/application/server,src/service/model -ot go -o src/application/server/swaggo-gen - + @echo " test - Run tests, database will be setup" + @echo " run - Run the application" + @echo " setup-db - Setup the database" + @echo " clear-db - Clear the database" + .PHONY: build -build: gen-proto gen-swagger +build: gen-swagger @echo "Building on $(OS)" go mod tidy go build -o artifacts/bin/migrate_db src/application/migrate_db/main.go go build -o artifacts/bin/service src/application/server/main.go - go build -o artifacts/bin/asynq_worker src/application/asynq_worker/main.go - go build -o artifacts/bin/rpc_server src/application/rpc_server/main.go go build -o artifacts/bin/schedule src/application/schedule/main.go +.PHONY: build-image +build-image: + docker build -f docker/oj-lab-platform.dockerfile -t oj-lab-platform:latest . + .PHONY: clear-db clear-db: docker-compose -f environment/docker-compose.yml -p oj-lab-platform stop @@ -57,6 +36,11 @@ setup-db: clear-db build sleep 10s ./artifacts/bin/migrate_db +.PHONY: gen-swagger +gen-swagger: install-swaggo + swag fmt -d src/application/server + swag init -d src/application/server,src/service/model -ot go -o src/application/server/swaggo-gen + .PHONY: check check: gen-proto go vet ./... @@ -65,10 +49,6 @@ check: gen-proto test: gen-swagger check setup-db go test -cover -v ./... -.PHONY: run-task-worker -run-task-worker: build check - ./artifacts/bin/asynq_worker - .PHONY: run-schedule run-schedule: build check ./artifacts/bin/schedule @@ -77,18 +57,31 @@ run-schedule: build check run-server: build check ./artifacts/bin/service -.PHONY: run-rpc-server -run-rpc-server: build check - ./artifacts/bin/rpc_server +.PHONY: run +run: build check + make -j run-server run-schedule + +.PHONY: get-front +get-front: + ./scripts/update-frontend-dist.sh artifacts/oj-lab-front/dist + +.PHONY: install-swaggo +install-swaggo: + go install github.com/swaggo/swag/cmd/swag@latest -.PHONY: run-background -run-background: build check - make -j run-schedule -.PHONY: run-all -run-all: build check - make -j run-server run-schedule +# Deprecated +.PHONY: gen-proto +gen-proto: install-proto + protoc --go_out=. --go_opt=paths=source_relative \ + --go-grpc_out=. --go-grpc_opt=paths=source_relative \ + src/service/proto/*.proto -.PHONY: build-docker -build-docker: - docker build -f docker/oj-lab-platform.dockerfile -t oj-lab-platform:latest . \ No newline at end of file +# Deprecated +.PHONY: install-proto +install-proto: + @# Referencing https://grpc.io/docs/protoc-installation/ + @./scripts/install-protoc.sh + @# Track https://grpc.io/docs/languages/go/quickstart/ for update + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 \ No newline at end of file diff --git a/go.mod b/go.mod index a573a53..815e2fa 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,6 @@ require ( github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/robfig/cron/v3 v3.0.1 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/rs/xid v1.5.0 // indirect github.com/spf13/afero v1.9.5 // indirect @@ -49,7 +48,6 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect golang.org/x/arch v0.4.0 // indirect - golang.org/x/time v0.1.0 // indirect golang.org/x/tools v0.7.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect google.golang.org/grpc v1.58.2 @@ -67,7 +65,6 @@ require ( github.com/go-playground/validator/v10 v10.15.1 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 - github.com/hibiken/asynq v0.24.1 github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jinzhu/inflection v1.0.0 // indirect diff --git a/go.sum b/go.sum index 906a896..a52d2aa 100644 --- a/go.sum +++ b/go.sum @@ -46,7 +46,6 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/alexedwards/argon2id v0.0.0-20230305115115-4b3c3280a736 h1:qZaEtLxnqY5mJ0fVKbk31NVhlgi0yrKm51Pq/I5wcz4= github.com/alexedwards/argon2id v0.0.0-20230305115115-4b3c3280a736/go.mod h1:mTeFRcTdnpzOlRjMoFYC/80HwVUreupyAiqPkCZQOXc= -github.com/bsm/ginkgo/v2 v2.7.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w= github.com/bsm/ginkgo/v2 v2.9.5 h1:rtVBYPs3+TC5iLUVOis1B9tjLTup7Cj5IfzosKtvTJ0= github.com/bsm/ginkgo/v2 v2.9.5/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= @@ -148,7 +147,6 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -163,7 +161,6 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -182,7 +179,6 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -192,8 +188,6 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hibiken/asynq v0.24.1 h1:+5iIEAyA9K/lcSPvx3qoPtsKJeKI5u9aOIvUmSsazEw= -github.com/hibiken/asynq v0.24.1/go.mod h1:u5qVeSbrnfT+vtG5Mq8ZPzQu/BmCKMHvTGb91uy9Tts= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -259,11 +253,8 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/redis/go-redis/v9 v9.0.3/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/redis/go-redis/v9 v9.1.0 h1:137FnGdk+EQdCbye1FW+qOEcY5S+SpY9T0NiuqvtfMY= github.com/redis/go-redis/v9 v9.1.0/go.mod h1:urWj3He21Dj5k4TK1y59xH8Uj6ATueP8AH1cY3lZl4c= -github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= -github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= @@ -273,7 +264,6 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= @@ -313,7 +303,6 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -321,8 +310,6 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.4.0 h1:A8WCeEWhLwPBKNbFi5Wv5UTCBx5zzubnXDlMOFAzFMc= golang.org/x/arch v0.4.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= @@ -370,7 +357,6 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= @@ -406,7 +392,6 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -434,7 +419,6 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -469,13 +453,10 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -503,8 +484,6 @@ golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= -golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -552,7 +531,6 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= @@ -659,7 +637,6 @@ google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/src/application/asynq_worker/main.go b/src/application/asynq_worker/main.go deleted file mode 100644 index 2037dff..0000000 --- a/src/application/asynq_worker/main.go +++ /dev/null @@ -1,16 +0,0 @@ -package main - -import ( - "github.com/OJ-lab/oj-lab-services/src/core" - asynqAgent "github.com/OJ-lab/oj-lab-services/src/core/agent/asynq" - "github.com/OJ-lab/oj-lab-services/src/service/business" -) - -func main() { - core.AppLogger().Info("Starting task server...") - config := asynqAgent.AsynqServerConfig{ - Concurrency: 10, - UsePriority: true, - } - asynqAgent.RunServer(config, business.GetAsynqMuxJudger()) -} diff --git a/src/application/rpc_server/main.go b/src/application/rpc_server/main.go index 7d3ded9..7bd2216 100644 --- a/src/application/rpc_server/main.go +++ b/src/application/rpc_server/main.go @@ -1,3 +1,5 @@ +// Deprecated currently +// Keep it for future use package main import ( diff --git a/src/core/agent/asynq/client.go b/src/core/agent/asynq/client.go deleted file mode 100644 index 855db23..0000000 --- a/src/core/agent/asynq/client.go +++ /dev/null @@ -1,28 +0,0 @@ -package asynqAgent - -import ( - "github.com/hibiken/asynq" -) - -type AsynqClient struct { - innerClient *asynq.Client -} - -var client *AsynqClient - -func GetDefaultTaskClient() *AsynqClient { - if client == nil { - client = &AsynqClient{ - innerClient: asynq.NewClient(asynq.RedisClientOpt{ - Addr: redisHost, - }), - } - } - - return client -} - -func (ac *AsynqClient) EnqueueTask(task *asynq.Task, opts ...asynq.Option) error { - _, err := ac.innerClient.Enqueue(task) - return err -} diff --git a/src/core/agent/asynq/middleware.go b/src/core/agent/asynq/middleware.go deleted file mode 100644 index 28b8023..0000000 --- a/src/core/agent/asynq/middleware.go +++ /dev/null @@ -1,22 +0,0 @@ -package asynqAgent - -import ( - "context" - "time" - - "github.com/OJ-lab/oj-lab-services/src/core" - "github.com/hibiken/asynq" -) - -func LoggingMiddleware(h asynq.Handler) asynq.Handler { - return asynq.HandlerFunc(func(ctx context.Context, t *asynq.Task) error { - start := time.Now() - core.AppLogger().Printf("Start processing %q", t.Type()) - err := h.ProcessTask(ctx, t) - if err != nil { - return err - } - core.AppLogger().Printf("Finished processing %q: Elapsed Time = %v", t.Type(), time.Since(start)) - return nil - }) -} diff --git a/src/core/agent/asynq/priority.go b/src/core/agent/asynq/priority.go deleted file mode 100644 index 8c074fb..0000000 --- a/src/core/agent/asynq/priority.go +++ /dev/null @@ -1,16 +0,0 @@ -package asynqAgent - -type AsynqTaskPriority string - -const ( - AsynqTaskPriorityCritical AsynqTaskPriority = "critical" - // Default priority ruled in asynq library - AsynqTaskPriorityDefault AsynqTaskPriority = "default" - AsynqTaskPriorityLow AsynqTaskPriority = "low" -) - -var priorityMap = map[string]int{ - string(AsynqTaskPriorityCritical): 6, - string(AsynqTaskPriorityDefault): 3, - string(AsynqTaskPriorityLow): 1, -} diff --git a/src/core/agent/asynq/redis.go b/src/core/agent/asynq/redis.go deleted file mode 100644 index ee687ee..0000000 --- a/src/core/agent/asynq/redis.go +++ /dev/null @@ -1,16 +0,0 @@ -package asynqAgent - -import "github.com/OJ-lab/oj-lab-services/src/core" - -const ( - redisHostProp = "redis.host" -) - -var redisHost string - -func init() { - redisHost = core.AppConfig.GetString(redisHostProp) - if redisHost == "" { - panic("redis addr is not set") - } -} diff --git a/src/core/agent/asynq/scheduler.go b/src/core/agent/asynq/scheduler.go deleted file mode 100644 index 17b16a0..0000000 --- a/src/core/agent/asynq/scheduler.go +++ /dev/null @@ -1,31 +0,0 @@ -package asynqAgent - -import ( - "github.com/OJ-lab/oj-lab-services/src/core" - "github.com/hibiken/asynq" -) - -type ScheduleTask struct { - Cronspec string - Task *asynq.Task -} - -func RunSecheduler(scheduleTasks ...ScheduleTask) { - scheduler := asynq.NewScheduler( - asynq.RedisClientOpt{Addr: redisHost}, - nil, - ) - - for _, scheduleTask := range scheduleTasks { - entryID, err := scheduler.Register(scheduleTask.Cronspec, scheduleTask.Task) - if err != nil { - panic(err) - } - core.AppLogger().Infof( - "setup schedule task: %s, cronspec: %s, entryID: %s", - scheduleTask.Task.Type(), scheduleTask.Cronspec, entryID, - ) - } - - scheduler.Run() -} diff --git a/src/core/agent/asynq/server.go b/src/core/agent/asynq/server.go deleted file mode 100644 index d56ca2a..0000000 --- a/src/core/agent/asynq/server.go +++ /dev/null @@ -1,40 +0,0 @@ -package asynqAgent - -import ( - "log" - - "github.com/hibiken/asynq" -) - -type AsynqMux struct { - Pattern string - *asynq.ServeMux -} - -type AsynqServerConfig struct { - Concurrency int - UsePriority bool -} - -func RunServer(config AsynqServerConfig, subMuxs ...AsynqMux) { - asynqConfig := asynq.Config{ - Concurrency: config.Concurrency, - } - if config.UsePriority { - asynqConfig.Queues = priorityMap - } - - server := asynq.NewServer( - asynq.RedisClientOpt{Addr: redisHost}, - asynqConfig, - ) - - mux := asynq.NewServeMux() - for _, subMux := range subMuxs { - mux.Handle(subMux.Pattern, subMux.ServeMux) - } - - if err := server.Run(mux); err != nil { - log.Fatalf("could not run server: %v", err) - } -} diff --git a/src/service/business/judger_task.go b/src/service/business/judger_task.go deleted file mode 100644 index cbca295..0000000 --- a/src/service/business/judger_task.go +++ /dev/null @@ -1,252 +0,0 @@ -// ! Deprected - -package business - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/OJ-lab/oj-lab-services/src/core" - asynqAgent "github.com/OJ-lab/oj-lab-services/src/core/agent/asynq" - gormAgent "github.com/OJ-lab/oj-lab-services/src/core/agent/gorm" - judgerAgent "github.com/OJ-lab/oj-lab-services/src/core/agent/judger" - "github.com/OJ-lab/oj-lab-services/src/service/mapper" - "github.com/OJ-lab/oj-lab-services/src/service/model" - "github.com/hibiken/asynq" - "gorm.io/gorm" - "gorm.io/gorm/clause" -) - -const ( - MuxPatternJudger = "judger" - TaskNameJudgerTrackAllState = "judger:track_all_state" - TaskNameJudgerGetState = "judger:get_state" - TaskNameJudgerHandleSubmission = "judger:handle_submission" -) - -func NewTaskJudgerTrackAllState() *asynq.Task { - return asynq.NewTask(TaskNameJudgerTrackAllState, nil) -} - -func NewTaskJudgerGetState(judger model.Judger) *asynq.Task { - judgerJson, err := json.Marshal(judger) - if err != nil { - panic(err) - } - return asynq.NewTask(TaskNameJudgerGetState, judgerJson) -} - -func GetAsynqMuxJudger() asynqAgent.AsynqMux { - serveMux := asynq.NewServeMux() - serveMux.Use(asynqAgent.LoggingMiddleware) - serveMux.HandleFunc(TaskNameJudgerTrackAllState, handleTaskJudgerTrackAllState) - serveMux.HandleFunc(TaskNameJudgerGetState, handleTaskJudgerGetState) - serveMux.HandleFunc(TaskNameJudgerHandleSubmission, handleTaskJudgerHandleSubmission) - - return asynqAgent.AsynqMux{ - Pattern: MuxPatternJudger, - ServeMux: serveMux, - } -} - -func handleTaskJudgerTrackAllState(ctx context.Context, task *asynq.Task) error { - db := gormAgent.GetDefaultDB() - judgerList, err := mapper.GetJudgerList(db) - if err != nil { - return err - } - core.AppLogger().Infof("judger list: %v", judgerList) - - asynqClient := asynqAgent.GetDefaultTaskClient() - for _, judger := range judgerList { - err := asynqClient.EnqueueTask( - NewTaskJudgerGetState(judger), - asynq.TaskID(fmt.Sprintf("%s:%s", TaskNameJudgerGetState, judger.Host)), - ) - if err != nil { - core.AppLogger().Errorf("failed to enqueue task %s: %v", TaskNameJudgerGetState, err) - } - } - - return nil -} - -func handleTaskJudgerGetState(ctx context.Context, task *asynq.Task) error { - db := gormAgent.GetDefaultDB() - var judger model.Judger - if err := json.Unmarshal(task.Payload(), &judger); err != nil { - return err - } - - err := db.Transaction(func(tx *gorm.DB) error { - if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("host = ?", judger.Host).First(&judger).Error; err != nil { - return err - } - - judgerClient := judgerAgent.JudgerClient{ - Host: judger.Host, - } - judgerStateString, err := judgerClient.GetState() - if err != nil { - return err - } - judgerState := model.StringToJudgerState(judgerStateString) - core.AppLogger().Debugf("Get Judger %v state=%v", judgerClient.Host, judgerState) - - if judgerState == model.JudgerStateIdle { - core.AppLogger().Debugf("Judger %v is idle, try find submission to handle", judgerClient.Host) - asynqClient := asynqAgent.GetDefaultTaskClient() - err := asynqClient.EnqueueTask( - NewTaskJudgerHandleSubmission(judger), - asynq.TaskID(fmt.Sprintf("%s:%s", TaskNameJudgerHandleSubmission, judger.Host)), - ) - if err != nil { - core.AppLogger().Errorf("failed to enqueue task %s: %v", TaskNameJudgerHandleSubmission, err) - } - } - - if !judger.State.CanUpdate(model.JudgerStateIdle) { - core.AppLogger().Debugf("Judger state is invalid or don't need update, ignoring this state update") - return nil - } - judger.State = judgerState - - err = tx.Model(&judger).Update("state", judgerState).Error - if err != nil { - return err - } - - return nil - }) - - if err != nil { - return err - } - return nil -} - -func NewTaskJudgerHandleSubmission(judger model.Judger) *asynq.Task { - judgerJson, err := json.Marshal(judger) - if err != nil { - panic(err) - } - return asynq.NewTask(TaskNameJudgerHandleSubmission, judgerJson) -} - -func handleTaskJudgerHandleSubmission(ctx context.Context, task *asynq.Task) error { - db := gormAgent.GetDefaultDB() - var judger model.Judger - if err := json.Unmarshal(task.Payload(), &judger); err != nil { - return err - } - - judgerClient := judgerAgent.JudgerClient{ - Host: judger.Host, - } - judgerStateString, err := judgerClient.GetState() - if err != nil { - return err - } - judgerState := model.StringToJudgerState(judgerStateString) - if judgerState != model.JudgerStateIdle { - core.AppLogger().Debugf("Judger %v is not idle, ignoring this task", judgerClient.Host) - return nil - } - - var submission model.JudgeTaskSubmission - err = db.Transaction(func(tx *gorm.DB) error { - if err := mapper.BuildGetSubmissionTXByOptions(tx.Clauses(clause.Locking{ - Strength: "UPDATE", - }), mapper.GetSubmissionOptions{ - Statuses: []model.SubmissionStatus{ - model.SubmissionStatusPending, - }, - Limit: func() *int { - limit := 1 - return &limit - }(), - OrderByColumns: []model.OrderByColumnOption{ - { - Column: "create_at", - Desc: true, - }, - }, - }, false).First(&submission).Error; err != nil { - core.AppLogger().Errorf("Failed to get submission: %v", err) - return err - } - core.AppLogger().Debugf("Get submission %v", submission.UID) - submission.Status = model.SubmissionStatusRunning - err = tx.Model(&submission).Update("status", submission.Status).Error - if err != nil { - core.AppLogger().Errorf("Failed to update submission status: %v", err) - return err - } - - return nil - }) - - if err != nil { - core.AppLogger().Errorf("Failed to select submission: %v", err) - return err - } - - err = db.Transaction(func(tx *gorm.DB) error { - if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("host = ?", judger.Host).First(&judger).Error; err != nil { - return err - } - judger.State = model.JudgerStateBusy - err = tx.Model(&judger).Update("state", judger.State).Error - if err != nil { - return err - } - - core.AppLogger().Debugf("Judger %v start to handle submission %v", judgerClient.Host, submission.UID) - judgeVerdict, err := judgerClient.PostJudgeSync( - submission.ProblemSlug, - judgerAgent.JudgeRequest{ - Language: string(submission.Language), - Code: submission.Code, - }, - ) - if err != nil { - core.AppLogger().Errorf("Failed to judge submission: %v", err) - return err - } - core.AppLogger().Debugf("Get Judger %v verdict=%v", judgerClient.Host, judgeVerdict) - - submission.Status = model.SubmissionStatusFinished - verdictBytes, err := json.Marshal(judgeVerdict) - if err != nil { - core.AppLogger().Errorf("Failed to marshal verdict: %v", err) - return err - } - err = tx.Model(&submission).Updates(map[string]interface{}{ - "status": submission.Status, - "verdict_json": string(verdictBytes), - }).Error - if err != nil { - return err - } - - judger.State = model.JudgerStateIdle - err = tx.Model(&judger).Update("state", judger.State).Error - if err != nil { - return err - } - - return nil - }) - - if err != nil { - core.AppLogger().Errorf("Failed to handle submission: %v", err) - err = db.Model(&submission).Update("status", model.SubmissionStatusPending).Error - if err != nil { - core.AppLogger().Errorf("failed to update submission status: %v", err) - } - - return err - } - return nil -} diff --git a/src/service/proto/oj_lab_greeting.proto b/src/service/proto/oj_lab_greeting.proto index 1208e37..8a4dec8 100644 --- a/src/service/proto/oj_lab_greeting.proto +++ b/src/service/proto/oj_lab_greeting.proto @@ -1,3 +1,5 @@ +// Deprecated currently +// Keep it for future use syntax = "proto3"; package oj_lab_greeting.protos; diff --git a/src/service/proto/oj_lab_stream.proto b/src/service/proto/oj_lab_stream.proto index 55e3903..c3e4f73 100644 --- a/src/service/proto/oj_lab_stream.proto +++ b/src/service/proto/oj_lab_stream.proto @@ -1,3 +1,5 @@ +// Deprecated currently +// Keep it for future use syntax = "proto3"; package oj_lab_stream.protos;