diff --git a/Dockerfile b/Dockerfile index 19fd962..535ea3f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,18 @@ -ARG egover=1.5.4 +ARG egover=1.6.0 ARG baseimage=ghcr.io/edgelesssys/ego-deploy:v${egover} ARG VERSION + # Build the Go binary in a separate stage utilizing Makefile -FROM ghcr.io/edgelesssys/ego-dev:v${egover} AS builder +FROM ghcr.io/edgelesssys/ego-dev:v${egover} AS dependencies WORKDIR /app COPY go.mod go.sum ./ RUN go mod download COPY . . +# Build the Go binary in a separate stage utilizing Makefile +FROM dependencies AS builder + ENV VERSION=${VERSION} RUN make build diff --git a/Makefile b/Makefile index ee0a2b0..d7718cb 100644 --- a/Makefile +++ b/Makefile @@ -34,5 +34,5 @@ docker-build: tee/private.pem @docker build --secret id=private_key,src=./tee/private.pem -t $(IMAGE) -f Dockerfile . test: tee/private.pem - @docker build --build-arg baseimage=builder --secret id=private_key,src=./tee/private.pem -t $(IMAGE) -f Dockerfile . + @docker build --target=dependencies --build-arg baseimage=builder --secret id=private_key,src=./tee/private.pem -t $(IMAGE) -f Dockerfile . @docker run --user root -v $(PWD)/coverage:/app/coverage --rm --workdir /app $(IMAGE) go test -coverprofile=coverage/coverage.txt -covermode=atomic -v ./... diff --git a/api/types/job.go b/api/types/job.go index 5715dfc..36ebce9 100644 --- a/api/types/job.go +++ b/api/types/job.go @@ -12,8 +12,8 @@ type JobResponse struct { type JobArguments map[string]interface{} type JobResult struct { - Error string `json:"error"` - Data interface{} `json:"data"` + Error string `json:"error"` + Data []byte `json:"data"` } func (jr JobResult) Success() bool { @@ -26,6 +26,10 @@ type Job struct { UUID string `json:"-"` } +func (jr JobResult) Unmarshal(i interface{}) error { + return json.Unmarshal(jr.Data, i) +} + func (ja JobArguments) Unmarshal(i interface{}) error { dat, err := json.Marshal(ja) if err != nil { diff --git a/go.mod b/go.mod index 0ff52f8..be02e8c 100644 --- a/go.mod +++ b/go.mod @@ -1,33 +1,54 @@ module github.com/masa-finance/tee-worker -go 1.22.2 +go 1.21 + +toolchain go1.21.11 require ( + github.com/cenkalti/backoff v2.2.1+incompatible github.com/edgelesssys/ego v1.5.4 + github.com/gocolly/colly v1.2.0 github.com/google/uuid v1.6.0 + github.com/joho/godotenv v1.5.1 github.com/labstack/echo/v4 v4.12.0 - github.com/onsi/ginkgo/v2 v2.20.2 - github.com/onsi/gomega v1.34.2 + github.com/onsi/ginkgo/v2 v2.15.0 + github.com/onsi/gomega v1.32.0 + github.com/sirupsen/logrus v1.9.3 ) require ( + github.com/PuerkitoBio/goquery v1.9.0 // indirect + github.com/andybalholm/cascadia v1.3.2 // indirect + github.com/antchfx/htmlquery v1.3.3 // indirect + github.com/antchfx/xmlquery v1.4.2 // indirect + github.com/antchfx/xpath v1.3.2 // indirect + github.com/chzyer/readline v1.5.1 // indirect github.com/go-jose/go-jose/v4 v4.0.4 // indirect github.com/go-logr/logr v1.4.2 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/gobwas/glob v0.2.3 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect - github.com/joho/godotenv v1.5.1 // indirect + github.com/google/pprof v0.0.0-20240525172833-67f7ab83a680 // indirect + github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465 // indirect + github.com/kennygrant/sanitize v1.2.4 // indirect github.com/labstack/gommon v0.4.2 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect + github.com/temoto/robotstxt v1.1.2 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect golang.org/x/crypto v0.27.0 // indirect - golang.org/x/net v0.28.0 // indirect + golang.org/x/net v0.29.0 // indirect golang.org/x/sys v0.25.0 // indirect golang.org/x/text v0.18.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.24.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/protobuf v1.34.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 1a19871..fbafb84 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,22 @@ +github.com/PuerkitoBio/goquery v1.9.0 h1:zgjKkdpRY9T97Q5DCtcXwfqkcylSFIVCocZmn2huTp8= +github.com/PuerkitoBio/goquery v1.9.0/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY= +github.com/PuerkitoBio/goquery v1.10.0 h1:6fiXdLuUvYs2OJSvNRqlNPoBm6YABE226xrbavY5Wv4= +github.com/PuerkitoBio/goquery v1.10.0/go.mod h1:TjZZl68Q3eGHNBA8CWaxAN7rOU1EbDz3CWuolcO5Yu4= +github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= +github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= +github.com/antchfx/htmlquery v1.3.3 h1:x6tVzrRhVNfECDaVxnZi1mEGrQg3mjE/rxbH2Pe6dNE= +github.com/antchfx/htmlquery v1.3.3/go.mod h1:WeU3N7/rL6mb6dCwtE30dURBnBieKDC/fR8t6X+cKjU= +github.com/antchfx/xmlquery v1.4.2 h1:MZKd9+wblwxfQ1zd1AdrTsqVaMjMCwow3IqkCSe00KA= +github.com/antchfx/xmlquery v1.4.2/go.mod h1:QXhvf5ldTuGqhd1SHNvvtlhhdQLks4dD0awIVhXIDTA= +github.com/antchfx/xpath v1.3.2 h1:LNjzlsSjinu3bQpw9hWMY9ocB80oLOWuQqFvO6xt51U= +github.com/antchfx/xpath v1.3.2/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= +github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI= +github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= +github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/edgelesssys/ego v1.5.4 h1:ADc6t5j77mOfwwu+akZX/I41YzHoseYiBcM5aME+Hb0= @@ -6,18 +25,38 @@ github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI= +github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20240525172833-67f7ab83a680 h1:jtWKMkehOTiort4R+HpL3BqB55Q8BhieGpN3eL1mPdo= +github.com/google/pprof v0.0.0-20240525172833-67f7ab83a680/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSFBy+X1V0o+l+8NF1avt4HWl7cA= github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465 h1:KwWnWVWCNtNq/ewIX7HIKnELmEx2nDP42yskD/pi7QE= +github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= +github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= github.com/labstack/echo/v4 v4.12.0 h1:IKpw49IMryVB2p1a4dzwlhP1O2Tf2E0Ir/450lH+kI0= github.com/labstack/echo/v4 v4.12.0/go.mod h1:UP9Cr2DJXbOK3Kr9ONYzNowSh7HP0aG0ShAyycHSJvM= github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= @@ -27,35 +66,93 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY= +github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= github.com/onsi/ginkgo/v2 v2.20.2 h1:7NVCeyIWROIAheY21RLS+3j2bb52W0W82tkberYytp4= github.com/onsi/ginkgo/v2 v2.20.2/go.mod h1:K9gyxPIlb+aIvnZ8bd9Ak+YP18w3APlR+5coaZoE2ag= +github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= +github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8= github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= +github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/api/api_test.go b/internal/api/api_test.go index ec36b16..19208bf 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -9,6 +9,7 @@ import ( "github.com/masa-finance/tee-worker/api/types" . "github.com/masa-finance/tee-worker/internal/api" + "github.com/masa-finance/tee-worker/internal/jobs" "github.com/masa-finance/tee-worker/pkg/client" ) @@ -38,30 +39,64 @@ var _ = Describe("API", func() { cancel() }) - It("should submit a job and get the correct result", func() { + It("should submit an invalid job, and fail because of the malformed URL. no results containing google", func() { // Step 1: Create the job request job := types.Job{ - Type: "web-scraper", + Type: jobs.WebScraperType, Arguments: map[string]interface{}{ "url": "google", }, } // Step 2: Submit the job - jobID, err := clientInstance.SubmitJob(job) + jobResult, err := clientInstance.SubmitJob(job) + Expect(err).NotTo(HaveOccurred()) + Expect(jobResult.UUID).NotTo(BeEmpty()) + + // Step 3: Wait for the job result + encryptedResult, err := jobResult.Get() + Expect(err).NotTo(HaveOccurred()) + Expect(encryptedResult).NotTo(BeEmpty()) + + // Step 4: Decrypt the result + decryptedResult, err := clientInstance.Decrypt(encryptedResult) + Expect(err).NotTo(HaveOccurred()) + Expect(decryptedResult).NotTo(BeEmpty()) + Expect(decryptedResult).NotTo(ContainSubstring("google")) + Expect(decryptedResult).To(ContainSubstring(`"pages":null`)) + }) + + It("should submit a job and get the correct result", func() { + // Step 1: Create the job request + job := types.Job{ + Type: jobs.WebScraperType, + Arguments: map[string]interface{}{ + "url": "https://google.com", + "depth": 1, + }, + } + + // Step 2: Submit the job + jobResult, err := clientInstance.SubmitJob(job) Expect(err).NotTo(HaveOccurred()) - Expect(jobID).NotTo(BeEmpty()) + Expect(jobResult).NotTo(BeNil()) // Step 3: Wait for the job result - encryptedResult, err := clientInstance.WaitForResult(jobID, 10, time.Second) + encryptedResult, err := jobResult.Get() Expect(err).NotTo(HaveOccurred()) Expect(encryptedResult).NotTo(BeEmpty()) // Step 4: Decrypt the result - decryptedResult, err := clientInstance.DecryptResult(encryptedResult) + decryptedResult, err := clientInstance.Decrypt(encryptedResult) Expect(err).NotTo(HaveOccurred()) + Expect(decryptedResult).NotTo(BeEmpty()) Expect(decryptedResult).To(ContainSubstring("google")) + + result, err := jobResult.GetDecrypted() + Expect(err).NotTo(HaveOccurred()) + Expect(result).NotTo(BeEmpty()) + Expect(result).To(ContainSubstring("google")) }) It("bubble up errors", func() { @@ -74,12 +109,15 @@ var _ = Describe("API", func() { } // Step 2: Submit the job - jobID, err := clientInstance.SubmitJob(job) + jobResult, err := clientInstance.SubmitJob(job) Expect(err).NotTo(HaveOccurred()) - Expect(jobID).NotTo(BeEmpty()) + Expect(jobResult).NotTo(BeNil()) + Expect(jobResult.UUID).NotTo(BeEmpty()) + + jobResult.SetMaxRetries(10) // Step 3: Wait for the job result (should fail) - encryptedResult, err := clientInstance.WaitForResult(jobID, 10, time.Second) + encryptedResult, err := jobResult.Get() Expect(err).To(HaveOccurred()) Expect(encryptedResult).To(BeEmpty()) }) diff --git a/internal/api/start.go b/internal/api/start.go index 000aacd..094ed08 100644 --- a/internal/api/start.go +++ b/internal/api/start.go @@ -3,7 +3,6 @@ package api import ( "context" "encoding/base64" - "encoding/json" "net/http" "github.com/labstack/echo/v4" @@ -68,11 +67,7 @@ func Start(ctx context.Context, listenAddress string, config types.JobConfigurat return c.JSON(http.StatusInternalServerError, types.JobError{Error: res.Error}) } - dat, err := json.Marshal(res.Data) - if err != nil { - return err - } - sealedData, err := tee.Seal(dat) + sealedData, err := tee.Seal(res.Data) if err != nil { return err } diff --git a/internal/jobs/webscraper.go b/internal/jobs/webscraper.go index ddf39f9..1abfc97 100644 --- a/internal/jobs/webscraper.go +++ b/internal/jobs/webscraper.go @@ -1,10 +1,17 @@ package jobs import ( + "encoding/json" "fmt" + "net/http" + "strconv" "strings" + "time" + "github.com/cenkalti/backoff" + "github.com/gocolly/colly" "github.com/masa-finance/tee-worker/api/types" + "github.com/sirupsen/logrus" ) const WebScraperType = "web-scraper" @@ -18,7 +25,8 @@ type WebScraperConfiguration struct { } type WebScraperArgs struct { - URL string `json:"url"` + URL string `json:"url"` + Depth int `json:"depth"` } func NewWebScraper(jc types.JobConfiguration) *WebScraper { @@ -39,9 +47,162 @@ func (ws *WebScraper) ExecuteJob(j types.Job) (types.JobResult, error) { } } + result, err := scrapeWeb([]string{args.URL}, args.Depth) + if err != nil { + return types.JobResult{Error: err.Error()}, err + } + // Do the web scraping here // For now, just return the URL return types.JobResult{ - Data: args.URL, + Data: result, }, nil } + +// Section represents a distinct part of a scraped webpage, typically defined by a heading. +// It contains a Title, representing the heading of the section, and Paragraphs, a slice of strings +// containing the text content found within that section. +type Section struct { + Title string `json:"title"` // Title is the heading text of the section. + Paragraphs []string `json:"paragraphs"` // Paragraphs contains all the text content of the section. + Images []string `json:"images"` // Images storing base64 - maybe!!? +} + +// CollectedData represents the aggregated result of the scraping process. +// It contains a slice of Section structs, each representing a distinct part of a scraped webpage. +type CollectedData struct { + Sections []Section `json:"sections"` // Sections is a collection of webpage sections that have been scraped. + Pages []string `json:"pages"` +} + +// scrapeWeb initiates the scraping process for the given list of URIs. +// It returns a CollectedData struct containing the scraped sections from each URI, +// and an error if any occurred during the scraping process. +// +// Parameters: +// - uri: []string - list of URLs to scrape +// - depth: int - depth of how many subpages to scrape +// +// Returns: +// - []byte - JSON representation of the collected data +// - error - any error that occurred during the scraping process +// +// Example usage: +// +// go func() { +// res, err := scraper.scrapeWeb([]string{"https://en.wikipedia.org/wiki/Maize"}, 5) +// if err != nil { +// logrus.WithError(err).Error("Error collecting data") +// return +// } +// logrus.WithField("result", string(res)).Info("Scraping completed") +// }() +func scrapeWeb(uri []string, depth int) ([]byte, error) { + // Set default depth to 1 if 0 is provided + if depth <= 0 { + depth = 1 + } + + var collectedData CollectedData + + c := colly.NewCollector( + colly.Async(true), // Enable asynchronous requests + colly.AllowURLRevisit(), + colly.IgnoreRobotsTxt(), + colly.MaxDepth(depth), + ) + + // Adjust the parallelism and delay based on your needs and server capacity + limitRule := colly.LimitRule{ + DomainGlob: "*", + Parallelism: 4, // Increased parallelism + Delay: 500 * time.Millisecond, // Reduced delay + } + if err := c.Limit(&limitRule); err != nil { + logrus.Errorf("[-] Unable to set scraper limit. Using default. Error: %v", err) + } + + // Increase the timeout slightly if necessary + c.SetRequestTimeout(240 * time.Second) // Increased to 4 minutes + + // Initialize a backoff strategy + backoffStrategy := backoff.NewExponentialBackOff() + + c.OnError(func(r *colly.Response, err error) { + if r.StatusCode == http.StatusTooManyRequests { + // Parse the Retry-After header (in seconds) + retryAfter, convErr := strconv.Atoi(r.Headers.Get("Retry-After")) + if convErr != nil { + // If not in seconds, it might be a date. Handle accordingly. + logrus.Debugf("[-] Retry-After: %s", r.Headers.Get("Retry-After")) + } + // Calculate the next delay + nextDelay := backoffStrategy.NextBackOff() + if retryAfter > 0 { + nextDelay = time.Duration(retryAfter) * time.Second + } + logrus.Warnf("[-] Rate limited. Retrying after %v", nextDelay) + time.Sleep(nextDelay) + // Retry the request + _ = r.Request.Retry() + } else { + logrus.Errorf("[-] Request URL: %s failed with error: %v", r.Request.URL, err) + } + }) + + c.OnHTML("h1, h2", func(e *colly.HTMLElement) { + // Directly append a new Section to collectedData.Sections + collectedData.Sections = append(collectedData.Sections, Section{Title: e.Text}) + }) + + c.OnHTML("p", func(e *colly.HTMLElement) { + // Check if there are any sections to append paragraphs to + if len(collectedData.Sections) > 0 { + // Get a reference to the last section + lastSection := &collectedData.Sections[len(collectedData.Sections)-1] + // Append the paragraph to the last section + // Check for duplicate paragraphs before appending + isDuplicate := false + for _, paragraph := range lastSection.Paragraphs { + if paragraph == e.Text { + isDuplicate = true + break + } + } + // Handle dupes + if !isDuplicate { + lastSection.Paragraphs = append(lastSection.Paragraphs, e.Text) + } + } + }) + + c.OnHTML("img", func(e *colly.HTMLElement) { + imageURL := e.Request.AbsoluteURL(e.Attr("src")) + if len(collectedData.Sections) > 0 { + lastSection := &collectedData.Sections[len(collectedData.Sections)-1] + lastSection.Images = append(lastSection.Images, imageURL) + } + }) + + c.OnHTML("a", func(e *colly.HTMLElement) { + pageURL := e.Request.AbsoluteURL(e.Attr("href")) + // Check if the URL protocol is supported (http or https) + if strings.HasPrefix(pageURL, "http://") || strings.HasPrefix(pageURL, "https://") { + collectedData.Pages = append(collectedData.Pages, pageURL) + _ = e.Request.Visit(pageURL) + } + }) + + for _, u := range uri { + err := c.Visit(u) + if err != nil { + return nil, err + } + } + + // Wait for all requests to finish + c.Wait() + + j, _ := json.Marshal(collectedData) + return j, nil +} diff --git a/internal/jobs/webscraper_test.go b/internal/jobs/webscraper_test.go index f5f33cb..f6f849a 100644 --- a/internal/jobs/webscraper_test.go +++ b/internal/jobs/webscraper_test.go @@ -9,18 +9,42 @@ import ( ) var _ = Describe("Webscraper", func() { - It("should fake scraping for now", func() { + It("should scrape now", func() { webScraper := NewWebScraper(types.JobConfiguration{}) res, err := webScraper.ExecuteJob(types.Job{ - Type: "web-scraper", + Type: WebScraperType, + Arguments: map[string]interface{}{ + "url": "https://www.google.com", + }, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Error).To(BeEmpty()) + + var scrapedData CollectedData + res.Unmarshal(&scrapedData) + Expect(err).NotTo(HaveOccurred()) + + Expect(len(scrapedData.Pages)).ToNot(BeZero()) + }) + + It("does not return data with invalid hosts", func() { + webScraper := NewWebScraper(types.JobConfiguration{}) + + res, err := webScraper.ExecuteJob(types.Job{ + Type: WebScraperType, Arguments: map[string]interface{}{ "url": "google", }, }) Expect(err).NotTo(HaveOccurred()) Expect(res.Error).To(BeEmpty()) - Expect(res.Data.(string)).To(Equal("google")) + + var scrapedData CollectedData + res.Unmarshal(&scrapedData) + Expect(err).NotTo(HaveOccurred()) + + Expect(len(scrapedData.Pages)).To(BeZero()) }) It("should allow to blacklist urls", func() { @@ -29,7 +53,7 @@ var _ = Describe("Webscraper", func() { }) res, err := webScraper.ExecuteJob(types.Job{ - Type: "web-scraper", + Type: WebScraperType, Arguments: map[string]interface{}{ "url": "google", }, diff --git a/internal/jobserver/jobserver_test.go b/internal/jobserver/jobserver_test.go index c4627ec..8d56756 100644 --- a/internal/jobserver/jobserver_test.go +++ b/internal/jobserver/jobserver_test.go @@ -33,7 +33,7 @@ var _ = Describe("Jobserver", func() { Eventually(func() bool { result, exists := jobserver.GetJobResult(uuid) - return exists && result.Error == "" && result.Data.(string) == "google" + return exists && result.Error == "" && string(result.Data) == "google" }, "5s").Should(Not(BeNil())) }) }) diff --git a/pkg/client/http.go b/pkg/client/http.go index 58c2d7f..c71652a 100644 --- a/pkg/client/http.go +++ b/pkg/client/http.go @@ -3,7 +3,6 @@ package client import ( "bytes" "encoding/json" - "errors" "fmt" "io" "net/http" @@ -26,65 +25,39 @@ func NewClient(baseURL string) *Client { } } -// SubmitJob submits a new job to the server and returns the job UID. -func (c *Client) SubmitJob(job types.Job) (string, error) { +// SubmitJob submits a new job to the server and returns the job result. +func (c *Client) SubmitJob(job types.Job) (*JobResult, error) { jobJSON, err := json.Marshal(job) if err != nil { - return "", fmt.Errorf("error marshaling job: %w", err) + return nil, fmt.Errorf("error marshaling job: %w", err) } resp, err := c.HTTPClient.Post(c.BaseURL+"/job", "application/json", bytes.NewBuffer(jobJSON)) if err != nil { - return "", fmt.Errorf("error sending POST request: %w", err) + return nil, fmt.Errorf("error sending POST request: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("error: received status code %d", resp.StatusCode) + return nil, fmt.Errorf("error: received status code %d", resp.StatusCode) } body, err := io.ReadAll(resp.Body) if err != nil { - return "", fmt.Errorf("error reading response body: %w", err) + return nil, fmt.Errorf("error reading response body: %w", err) } var jobResp types.JobResponse err = json.Unmarshal(body, &jobResp) if err != nil { - return "", fmt.Errorf("error unmarshaling response: %w", err) + return nil, fmt.Errorf("error unmarshaling response: %w", err) } - return jobResp.UID, nil + return &JobResult{UUID: jobResp.UID, client: c, maxRetries: 60, delay: 1 * time.Second}, nil } -// GetJobResult retrieves the encrypted result of a job. -func (c *Client) GetJobResult(jobID string) (string, error) { - resp, err := c.HTTPClient.Get(c.BaseURL + "/job/" + jobID) - if err != nil { - return "", fmt.Errorf("error sending GET request: %w", err) - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", fmt.Errorf("error reading response body: %w", err) - } - - if resp.StatusCode == http.StatusNotFound { - err = fmt.Errorf("job not found or not ready") - } - - if resp.StatusCode != http.StatusOK { - respErr := types.JobError{} - json.Unmarshal(body, &respErr) - err = fmt.Errorf("error: %s", respErr.Error) - } - - return string(body), err -} - -// DecryptResult sends the encrypted result to the server to decrypt it. -func (c *Client) DecryptResult(encryptedResult string) (string, error) { +// Decrypt sends the encrypted result to the server to decrypt it. +func (c *Client) Decrypt(encryptedResult string) (string, error) { decryptReq := types.EncryptedRequest{ EncryptedResult: encryptedResult, } @@ -112,21 +85,28 @@ func (c *Client) DecryptResult(encryptedResult string) (string, error) { return string(body), nil } -// WaitForResult polls the server until the job result is ready or a timeout occurs. -func (c *Client) WaitForResult(jobID string, maxRetries int, delay time.Duration) (result string, err error) { - retries := 0 +// GetJobResult retrieves the encrypted result of a job. +func (c *Client) GetResult(jobUUID string) (string, error) { + resp, err := c.HTTPClient.Get(c.BaseURL + "/job/" + jobUUID) + if err != nil { + return "", fmt.Errorf("error sending GET request: %w", err) + } + defer resp.Body.Close() - for { - if retries >= maxRetries { - return "", errors.New("max retries reached") - } - retries++ + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("error reading response body: %w", err) + } + + if resp.StatusCode == http.StatusNotFound { + err = fmt.Errorf("job not found or not ready") + } - result, err = c.GetJobResult(jobID) - if err == nil { - break - } + if resp.StatusCode != http.StatusOK { + respErr := types.JobError{} + json.Unmarshal(body, &respErr) + err = fmt.Errorf("error: %s", respErr.Error) } - return + return string(body), err } diff --git a/pkg/client/http_test.go b/pkg/client/http_test.go index 02762d1..5f29932 100644 --- a/pkg/client/http_test.go +++ b/pkg/client/http_test.go @@ -4,7 +4,6 @@ import ( "encoding/json" "net/http" "net/http/httptest" - "time" "github.com/masa-finance/tee-worker/pkg/client" . "github.com/masa-finance/tee-worker/pkg/client" @@ -50,9 +49,9 @@ var _ = Describe("Client", func() { }) It("should submit a job successfully", func() { - uid, err := c.SubmitJob(mockJob) + jobResult, err := c.SubmitJob(mockJob) Expect(err).NotTo(HaveOccurred()) - Expect(uid).To(Equal(mockJobUID)) + Expect(jobResult.UUID).To(Equal(mockJobUID)) }) It("should return an error on HTTP failure", func() { @@ -62,32 +61,6 @@ var _ = Describe("Client", func() { }) }) - Context("GetJobResult", func() { - BeforeEach(func() { - server = setupServer(http.StatusOK, "encrypted-result") - c = client.NewClient(server.URL) - }) - - AfterEach(func() { - server.Close() - }) - - It("should retrieve the job result successfully", func() { - result, err := c.GetJobResult("job1") - Expect(err).NotTo(HaveOccurred()) - Expect(result).To(Equal("\"encrypted-result\"\n")) - }) - - It("should return an error when job is not found", func() { - server.Close() - server = setupServer(http.StatusNotFound, types.JobError{Error: "job not found"}) - c = client.NewClient(server.URL) - _, err := c.GetJobResult("invalid-job-id") - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("job not found")) - }) - }) - Context("DecryptResult", func() { BeforeEach(func() { server = setupServer(http.StatusOK, "decrypted-result") @@ -99,7 +72,7 @@ var _ = Describe("Client", func() { }) It("should decrypt result successfully", func() { - result, err := c.DecryptResult("encrypted-result") + result, err := c.Decrypt("encrypted-result") Expect(err).NotTo(HaveOccurred()) Expect(result).To(Equal("\"decrypted-result\"\n")) }) @@ -108,33 +81,8 @@ var _ = Describe("Client", func() { server.Close() server = setupServer(http.StatusInternalServerError, nil) c = client.NewClient(server.URL) - _, err := c.DecryptResult("encrypted-result") - Expect(err).To(HaveOccurred()) - }) - }) - - Context("WaitForResult", func() { - BeforeEach(func() { - server = setupServer(http.StatusOK, "encrypted-result") - c = client.NewClient(server.URL) - }) - - AfterEach(func() { - server.Close() - }) - - It("should wait and retrieve result successfully", func() { - result, err := c.WaitForResult("job1", 3, time.Millisecond*10) - Expect(err).NotTo(HaveOccurred()) - Expect(result).To(Equal("\"encrypted-result\"\n")) - }) - - It("should fail after max retries", func() { - server.Close() // simulate unavailability - result, err := c.WaitForResult("job1", 3, time.Millisecond*10) + _, err := c.Decrypt("encrypted-result") Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(Equal("max retries reached")) - Expect(result).To(BeEmpty()) }) }) }) diff --git a/pkg/client/result.go b/pkg/client/result.go new file mode 100644 index 0000000..40f6d45 --- /dev/null +++ b/pkg/client/result.go @@ -0,0 +1,56 @@ +package client + +import ( + "fmt" + "time" +) + +type JobResult struct { + UUID string + maxRetries int + delay time.Duration + client *Client +} + +func (jr *JobResult) SetMaxRetries(maxRetries int) { + jr.maxRetries = maxRetries +} + +func (jr *JobResult) SetDelay(delay time.Duration) { + jr.delay = delay +} + +// GetJobResult retrieves the encrypted result of a job. +func (jr *JobResult) getResult() (string, error) { + return jr.client.GetResult(jr.UUID) +} + +// Get polls the server until the job result is ready or a timeout occurs. +func (jr *JobResult) Get() (result string, err error) { + retries := 0 + + for { + if retries >= jr.maxRetries { + return "", fmt.Errorf("max retries reached: %w", err) + } + retries++ + + result, err = jr.getResult() + if err == nil { + break + } + time.Sleep(jr.delay) + } + + return +} + +// Get polls the server until the job result is ready or a timeout occurs. +func (jr *JobResult) GetDecrypted() (result string, err error) { + result, err = jr.Get() + if err == nil { + result, err = jr.client.Decrypt(result) + } + + return +}