diff --git a/.github/workflows/testingci.yml b/.github/workflows/testingci.yml index 3066a9f5..f4f5ba0f 100644 --- a/.github/workflows/testingci.yml +++ b/.github/workflows/testingci.yml @@ -33,6 +33,16 @@ jobs: runs-on: ubuntu-latest permissions: contents: read + services: + redis: + image: redis + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 @@ -41,8 +51,14 @@ jobs: - name: Generate go code from go:generate comments run: make setup # test - - name: Test units (w/o engines) + - name: Test units (w/o engines & redis) run: make test + # test-redis + - name: Test units (only redis) + run: make test-redis + env: + REDIS_HOST: localhost + REDIS_PORT: "6379" test-engines: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 39dbb38f..fb8e7cee 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,7 @@ profiling/ *_stringer.go *_enumer.go *_searcher.go + +# test dump + +testdump* \ No newline at end of file diff --git a/Makefile b/Makefile index 34fa2c93..a51efab5 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,15 @@ test: test-engines: sh ./scripts/test-engines.sh +test-redis: + sh ./scripts/test-redis.sh + +test-redis-podman: + sh ./scripts/test-redis-podman.sh + +test-redis-docker: + sh ./scripts/test-redis-docker.sh + update: go get -u ./... go mod tidy diff --git a/go.mod b/go.mod index bcef1446..f2a4ee9d 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.21.3 require ( github.com/alecthomas/kong v0.8.1 - github.com/cockroachdb/pebble v1.1.0 + github.com/dgraph-io/badger/v4 v4.2.0 github.com/fxamacker/cbor/v2 v2.5.0 github.com/gin-contrib/cors v1.5.0 github.com/gin-contrib/graceful v0.1.0 @@ -28,38 +28,35 @@ require ( ) require ( - github.com/DataDog/zstd v1.5.5 // indirect - github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.11.0 // indirect + github.com/BurntSushi/toml v1.2.1 // indirect + github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/bytedance/sonic v1.10.2 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/chenzhuoyu/iasm v0.9.1 // indirect - github.com/cockroachdb/errors v1.11.1 // indirect - github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect - github.com/cockroachdb/redact v1.1.5 // indirect - github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/felixge/fgprof v0.9.3 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect - github.com/getsentry/sentry-go v0.25.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.17.0 // indirect + github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/glog v1.2.0 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/pprof v0.0.0-20231205033806-a5a03c77bf08 // indirect + github.com/google/flatbuffers v23.5.26+incompatible // indirect + github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.4 // indirect + github.com/klauspost/compress v1.17.6 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect github.com/kr/pretty v0.3.1 // indirect - github.com/kr/text v0.2.0 // indirect - github.com/leodido/go-urn v1.3.0 // indirect - github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -67,18 +64,14 @@ require ( github.com/nlnwa/whatwg-url v0.4.0 // indirect github.com/pelletier/go-toml/v2 v2.1.1 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.17.0 // indirect - github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.45.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/x448/float16 v0.8.4 // indirect + go.opencensus.io v0.24.0 // indirect golang.org/x/arch v0.7.0 // indirect - golang.org/x/crypto v0.18.0 // indirect - golang.org/x/exp v0.0.0-20231127185646-65229373498e // indirect - golang.org/x/mod v0.14.0 // indirect + golang.org/x/crypto v0.19.0 // indirect + golang.org/x/mod v0.15.0 // indirect golang.org/x/sync v0.6.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect @@ -99,8 +92,8 @@ require ( github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/sourcegraph/conc v0.3.0 github.com/temoto/robotstxt v1.1.2 // indirect - golang.org/x/net v0.20.0 - golang.org/x/sys v0.16.0 // indirect + golang.org/x/net v0.21.0 + golang.org/x/sys v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.32.0 // indirect diff --git a/go.sum b/go.sum index e03a11a7..2c7ca4ef 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= -github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= @@ -29,12 +27,10 @@ github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwq github.com/antchfx/xpath v1.2.4/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/antchfx/xpath v1.2.5 h1:hqZ+wtQ+KIOV/S3bGZcIhpgYC26um2bZYP2KVGcR7VY= github.com/antchfx/xpath v1.2.5/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.2.2-0.20220111210104-dfa3e347c392/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/bits-and-blooms/bitset v1.11.0 h1:RMyy2mBBShArUAhfVRZJ2xyBO58KCBCtZFShw3umo6k= -github.com/bits-and-blooms/bitset v1.11.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= +github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -44,6 +40,7 @@ github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE= github.com/bytedance/sonic v1.10.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= @@ -57,26 +54,26 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= -github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= -github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8= -github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw= -github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= -github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v1.1.0 h1:pcFh8CdCIt2kmEpK0OIatq67Ln9uGDYY3d5XnE0LJG4= -github.com/cockroachdb/pebble v1.1.0/go.mod h1:sEHm5NOXxyiAoKWhoFxT8xMgd/f3RA6qUqQ1BXKrh2E= -github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= -github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= -github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= -github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= +github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= @@ -88,8 +85,6 @@ github.com/fxamacker/cbor/v2 v2.5.0 h1:oHsG0V/Q6E/wqTS2O1Cozzsy69nqCiguo5Q1a1ADi github.com/fxamacker/cbor/v2 v2.5.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= -github.com/getsentry/sentry-go v0.25.0 h1:q6Eo+hS+yoJlTO3uu/azhQadsD8V+jQn2D8VvX1eOyI= -github.com/getsentry/sentry-go v0.25.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/gin-contrib/cors v1.5.0 h1:DgGKV7DDoOn36DFkNtbHrjoRiT5ExCe+PC9/xp7aKvk= github.com/gin-contrib/cors v1.5.0/go.mod h1:TvU7MAZ3EwrPLI2ztzTt3tqgvBCq+wn8WpZmfADjupI= github.com/gin-contrib/graceful v0.1.0 h1:kBWTKBx+xwYTS9Dklyps+S45zJAHdezlXzUnPsRW2PE= @@ -101,8 +96,6 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= -github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= -github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= @@ -115,6 +108,8 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91 github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-playground/validator/v10 v10.17.0 h1:SmVVlfAOtlZncTxRuinDPomC2DkXJ4E5T9gDA0AIH74= github.com/go-playground/validator/v10 v10.17.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 h1:TQcrn6Wq+sKGkpyPvppOz99zsMBaUOKXq6HSv655U1c= +github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= @@ -126,6 +121,8 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= +github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -140,23 +137,29 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= +github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= -github.com/google/pprof v0.0.0-20231205033806-a5a03c77bf08 h1:PxlBVtIFHR/mtWk2i0gTEdCz+jBnqiuHNSki0epDbVs= -github.com/google/pprof v0.0.0-20231205033806-a5a03c77bf08/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 h1:E/LAvt58di64hlYjx7AsNS6C/ysHWYo+2qPCZKTQhRo= +github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hearchco/logger v1.0.0 h1:udap9hyVgFgeGlB8G6VWddjsZAyUqdkaurJKbflpqeQ= github.com/hearchco/logger v1.0.0/go.mod h1:e4BYNMrs0h97r5O8+ffvvOYGmuCBQ1l25F1r+7AOYSg= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= @@ -169,8 +172,8 @@ github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8Nz github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= +github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -197,8 +200,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= -github.com/leodido/go-urn v1.3.0 h1:jX8FDLfW4ThVXctBNZ+3cIWnCSnrACDV73r76dy0aQQ= -github.com/leodido/go-urn v1.3.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= @@ -206,8 +209,6 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= @@ -225,8 +226,6 @@ github.com/nlnwa/whatwg-url v0.4.0/go.mod h1:pLzpJjFPtA+n7RCLvp0GBxvDHa/2ckNCBK9 github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= -github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= -github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -234,15 +233,7 @@ github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA= github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= -github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= -github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= -github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk= github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -263,6 +254,7 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -284,6 +276,8 @@ github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcY github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc= golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= @@ -293,11 +287,9 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No= -golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -305,8 +297,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -320,6 +312,7 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -330,8 +323,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -356,6 +349,7 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -363,8 +357,8 @@ golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= @@ -410,7 +404,9 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -420,6 +416,7 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= @@ -432,6 +429,7 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/scripts/test-redis-docker.sh b/scripts/test-redis-docker.sh new file mode 100644 index 00000000..bdb1cea7 --- /dev/null +++ b/scripts/test-redis-docker.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +docker run --rm --name hearchco-redis -d -p 6379:6379 docker.io/library/redis && \ +go test $(go list ./... | grep /redis) -count=1 +docker stop hearchco-redis \ No newline at end of file diff --git a/scripts/test-redis-podman.sh b/scripts/test-redis-podman.sh new file mode 100644 index 00000000..e759ab2a --- /dev/null +++ b/scripts/test-redis-podman.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +podman run --rm --name hearchco-redis -d -p 6379:6379 docker.io/library/redis && \ +go test $(go list ./... | grep /redis) -count=1 +podman stop hearchco-redis \ No newline at end of file diff --git a/scripts/test-redis.sh b/scripts/test-redis.sh new file mode 100644 index 00000000..09c9ef10 --- /dev/null +++ b/scripts/test-redis.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +go test $(go list ./... | grep /redis) -count=1 diff --git a/scripts/test.sh b/scripts/test.sh index a8f5fe77..73cdf4f7 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -go test $(go list ./... | grep -v /engines/) +go test $(go list ./... | grep -v /engines/ | grep -v /redis) diff --git a/src/cache/badger/badger.go b/src/cache/badger/badger.go new file mode 100644 index 00000000..8c6770a2 --- /dev/null +++ b/src/cache/badger/badger.go @@ -0,0 +1,163 @@ +package badger + +import ( + "fmt" + "path" + "time" + + "github.com/dgraph-io/badger/v4" + "github.com/fxamacker/cbor/v2" + "github.com/hearchco/hearchco/src/anonymize" + "github.com/hearchco/hearchco/src/cache" + "github.com/hearchco/hearchco/src/config" + "github.com/rs/zerolog/log" +) + +type DB struct { + bdb *badger.DB +} + +func New(dataDirPath string, config config.Badger) (*DB, error) { + badgerPath := path.Join(dataDirPath, "database") + + var opt badger.Options + if config.Persist { + opt = badger.DefaultOptions(badgerPath).WithLoggingLevel(badger.WARNING) + } else { + opt = badger.DefaultOptions("").WithInMemory(true).WithLoggingLevel(badger.WARNING) + } + + bdb, err := badger.Open(opt) + + if err != nil { + log.Error(). + Err(err). + Bool("persistence", config.Persist). + Str("path", badgerPath). + Msg("badger.New(): error opening badger") + } else if config.Persist { + log.Info(). + Bool("persistence", config.Persist). + Str("path", badgerPath). + Msg("Successfully opened badger") + } else { + log.Info(). + Bool("persistence", config.Persist). + Msg("Successfully opened in-memory badger") + } + + return &DB{bdb: bdb}, err +} + +func (db *DB) Close() { + if err := db.bdb.Close(); err != nil { + log.Error().Err(err).Msg("badger.Close(): error closing badger") + } else { + log.Debug().Msg("Successfully closed badger") + } +} + +func (db *DB) Set(k string, v cache.Value, ttl ...time.Duration) error { + log.Debug().Msg("Caching...") + cacheTimer := time.Now() + + var setTtl time.Duration = 0 + if len(ttl) > 0 { + setTtl = ttl[0] + } + + if val, err := cbor.Marshal(v); err != nil { + return fmt.Errorf("badger.Set(): error marshaling value: %w", err) + } else if err := db.bdb.Update(func(txn *badger.Txn) error { + var e *badger.Entry + if setTtl != 0 { + e = badger.NewEntry([]byte(anonymize.HashToSHA256B64(k)), val).WithTTL(ttl[0]) + } else { + e = badger.NewEntry([]byte(anonymize.HashToSHA256B64(k)), val) + } + return txn.SetEntry(e) + // ^returns error into else if + }); err != nil { + return fmt.Errorf("badger.Set(): error setting KV to badger: %w", err) + } else { + cacheTimeSince := time.Since(cacheTimer) + log.Trace(). + Int64("ms", cacheTimeSince.Milliseconds()). + Int64("ns", cacheTimeSince.Nanoseconds()). + Msg("Cached results") + } + + return nil +} + +func (db *DB) Get(k string, o cache.Value, hashed ...bool) error { + var kInput string + if len(hashed) > 0 && hashed[0] { + kInput = k + } else { + kInput = anonymize.HashToSHA256B64(k) + } + + var val []byte + err := db.bdb.View(func(txn *badger.Txn) error { + item, err := txn.Get([]byte(kInput)) + if err != nil { + return err + } + + v, err := item.ValueCopy(nil) + val = v + + return err + }) + + if err == badger.ErrKeyNotFound { + log.Trace(). + Str("key", kInput). + Msg("Found no value in badger") + } else if err != nil { + return fmt.Errorf("badger.Get(): error getting value from badger for key %v: %w", kInput, err) + } else if err := cbor.Unmarshal(val, o); err != nil { + return fmt.Errorf("badger.Get(): failed unmarshaling value from badger for key %v: %w", kInput, err) + } + + return nil +} + +// returns time until the key expires, not the time it will be considered expired +func (db *DB) GetTTL(k string, hashed ...bool) (time.Duration, error) { + var kInput string + if len(hashed) > 0 && hashed[0] { + kInput = k + } else { + kInput = anonymize.HashToSHA256B64(k) + } + + var expiresIn time.Duration + err := db.bdb.View(func(txn *badger.Txn) error { + item, err := txn.Get([]byte(kInput)) + if err != nil { + return err + } + + expiresAtUnix := time.Unix(int64(item.ExpiresAt()), 0) + expiresIn = time.Until(expiresAtUnix) + + // returns negative time.Since() if expiresAtUnix is in the past + if expiresIn < 0 { + expiresIn = 0 + } + + return err + }) + + if err == badger.ErrKeyNotFound { + log.Trace(). + Str("key", kInput). + Msg("Found no value in badger") + } else if err != nil { + return expiresIn, fmt.Errorf("badger.Get(): error getting value from badger for key %v: %w", kInput, err) + } + + return expiresIn, nil +} diff --git a/src/cache/badger/badger_test.go b/src/cache/badger/badger_test.go new file mode 100644 index 00000000..1732684b --- /dev/null +++ b/src/cache/badger/badger_test.go @@ -0,0 +1,245 @@ +package badger_test + +import ( + "testing" + "time" + + badgerog "github.com/dgraph-io/badger/v4" + "github.com/hearchco/hearchco/src/cache/badger" + "github.com/hearchco/hearchco/src/config" +) + +func TestNewInMemory(t *testing.T) { + _, err := badger.New("", config.Badger{Persist: false}) + if err != nil { + t.Errorf("error opening in-memory badger: %v", err) + } +} + +func TestNewPersistence(t *testing.T) { + path := "./testdump/new" + _, err := badger.New(path, config.Badger{Persist: true}) + if err != nil { + t.Errorf("error opening badger at %v: %v", path, err) + } +} + +func TestCloseInMemory(t *testing.T) { + db, err := badger.New("", config.Badger{Persist: false}) + if err != nil { + t.Errorf("error opening in-memory badger: %v", err) + } + + db.Close() +} + +func TestClosePersistence(t *testing.T) { + path := "./testdump/close" + db, err := badger.New(path, config.Badger{Persist: true}) + if err != nil { + t.Errorf("error opening badger at %v: %v", path, err) + } + + db.Close() +} + +func TestSetInMemory(t *testing.T) { + db, err := badger.New("", config.Badger{Persist: false}) + if err != nil { + t.Errorf("error opening in-memory badger: %v", err) + } + + defer db.Close() + + err = db.Set("testkey", "testvalue") + if err != nil { + t.Errorf("error setting key-value pair: %v", err) + } +} + +func TestSetPersistence(t *testing.T) { + path := "./testdump/set" + db, err := badger.New(path, config.Badger{Persist: true}) + if err != nil { + t.Errorf("error opening badger at %v: %v", path, err) + } + + defer db.Close() + + err = db.Set("testkey", "testvalue") + if err != nil { + t.Errorf("error setting key-value pair: %v", err) + } +} + +func TestSetTTLInMemory(t *testing.T) { + db, err := badger.New("", config.Badger{Persist: false}) + if err != nil { + t.Errorf("error opening in-memory badger: %v", err) + } + + defer db.Close() + + err = db.Set("testkey", "testvalue", 100*time.Second) + if err != nil { + t.Errorf("error setting key-value pair with TTL: %v", err) + } +} + +func TestSetTTLPersistence(t *testing.T) { + path := "./testdump/setttl" + db, err := badger.New(path, config.Badger{Persist: true}) + if err != nil { + t.Errorf("error opening badger at %v: %v", path, err) + } + + defer db.Close() + + err = db.Set("testkey", "testvalue", 100*time.Second) + if err != nil { + t.Errorf("error setting key-value pair with TTL: %v", err) + } +} + +func TestGetInMemory(t *testing.T) { + db, err := badger.New("", config.Badger{Persist: false}) + if err != nil { + t.Errorf("error opening in-memory badger: %v", err) + } + + defer db.Close() + + err = db.Set("testkey", "testvalue") + if err != nil { + t.Errorf("error setting key-value pair: %v", err) + } + + var value string + err = db.Get("testkey", &value) + if err != nil { + t.Errorf("error getting key-value pair: %v", err) + } + + if value != "testvalue" { + t.Errorf("expected value: testvalue, got: %v", value) + } +} + +func TestGetPersistence(t *testing.T) { + path := "./testdump/get" + db, err := badger.New(path, config.Badger{Persist: true}) + if err != nil { + t.Errorf("error opening badger at %v: %v", path, err) + } + + defer db.Close() + + err = db.Set("testkey", "testvalue") + if err != nil { + t.Errorf("error setting key-value pair: %v", err) + } + + var value string + err = db.Get("testkey", &value) + if err != nil { + t.Errorf("error getting key-value pair: %v", err) + } + + if value != "testvalue" { + t.Errorf("expected value: testvalue, got: %v", value) + } +} + +func TestGetTTLInMemory(t *testing.T) { + db, err := badger.New("", config.Badger{Persist: false}) + if err != nil { + t.Errorf("error opening in-memory badger: %v", err) + } + + defer db.Close() + + err = db.Set("testkey", "testvalue", 100*time.Second) + if err != nil { + t.Errorf("error setting key-value pair with TTL: %v", err) + } + + ttl, err := db.GetTTL("testkey") + if err != nil { + t.Errorf("error getting TTL: %v", err) + } + + // TTL is not exact, so we check for a range + if ttl > 100*time.Second || ttl < 99*time.Second { + t.Errorf("expected 100s >= ttl >= 99s, got: %v", ttl) + } +} + +func TestGetTTLPersistence(t *testing.T) { + path := "./testdump/getttl" + db, err := badger.New(path, config.Badger{Persist: true}) + if err != nil { + t.Errorf("error opening badger at %v: %v", path, err) + } + + defer db.Close() + + err = db.Set("testkey", "testvalue", 100*time.Second) + if err != nil { + t.Errorf("error setting key-value pair with TTL: %v", err) + } + + ttl, err := db.GetTTL("testkey") + if err != nil { + t.Errorf("error getting TTL: %v", err) + } + + // TTL is not exact, so we check for a range + if ttl > 100*time.Second || ttl < 99*time.Second { + t.Errorf("expected 100s >= ttl >= 99s, got: %v", ttl) + } +} + +func TestGetInMemoryExpired(t *testing.T) { + db, err := badger.New("", config.Badger{Persist: false}) + if err != nil { + t.Errorf("error opening in-memory badger: %v", err) + } + + defer db.Close() + + err = db.Set("testkey", "testvalue", 1*time.Second) + if err != nil { + t.Errorf("error setting key-value pair with TTL: %v", err) + } + + time.Sleep(1 * time.Second) + + var value string + err = db.Get("testkey", &value) + if err != nil && err != badgerog.ErrKeyNotFound { + t.Errorf("error getting key-value pair: %v", err) + } +} + +func TestGetPersistenceExpired(t *testing.T) { + path := "./testdump/getexpired" + db, err := badger.New(path, config.Badger{Persist: true}) + if err != nil { + t.Errorf("error opening badger at %v: %v", path, err) + } + + defer db.Close() + + err = db.Set("testkey", "testvalue", 1*time.Second) + if err != nil { + t.Errorf("error setting key-value pair with TTL: %v", err) + } + + time.Sleep(1 * time.Second) + + var value string + err = db.Get("testkey", &value) + if err != nil && err != badgerog.ErrKeyNotFound { + t.Errorf("error getting key-value pair: %v", err) + } +} diff --git a/src/cache/interfaces.go b/src/cache/interfaces.go index 7fadcafb..17f48475 100644 --- a/src/cache/interfaces.go +++ b/src/cache/interfaces.go @@ -1,9 +1,12 @@ package cache +import "time" + type DB interface { Close() - Set(k string, v Value) error - Get(k string, o Value) error + Set(k string, v Value, ttl ...time.Duration) error + Get(k string, o Value, hashed ...bool) error + GetTTL(k string, hashed ...bool) (time.Duration, error) } type Value interface{} diff --git a/src/cache/nocache/nocache.go b/src/cache/nocache/nocache.go index eed8b7b1..2ec5e05b 100644 --- a/src/cache/nocache/nocache.go +++ b/src/cache/nocache/nocache.go @@ -1,15 +1,19 @@ package nocache import ( + "time" + "github.com/hearchco/hearchco/src/cache" ) type DB struct{} -func New() *DB { return nil } +func New() (*DB, error) { return nil, nil } func (db *DB) Close() {} -func (db *DB) Set(k string, v cache.Value) error { return nil } +func (db *DB) Set(k string, v cache.Value, ttl ...time.Duration) error { return nil } + +func (db *DB) Get(k string, o cache.Value, hashed ...bool) error { return nil } -func (db *DB) Get(k string, o cache.Value) error { return nil } +func (db *DB) GetTTL(k string, hashed ...bool) (time.Duration, error) { return 0, nil } diff --git a/src/cache/nocache/nocache_test.go b/src/cache/nocache/nocache_test.go new file mode 100644 index 00000000..491abc45 --- /dev/null +++ b/src/cache/nocache/nocache_test.go @@ -0,0 +1,98 @@ +package nocache_test + +import ( + "testing" + + "github.com/hearchco/hearchco/src/cache/nocache" +) + +func TestNew(t *testing.T) { + _, err := nocache.New() + if err != nil { + t.Errorf("error creating nocache: %v", err) + } +} + +func TestClose(t *testing.T) { + db, err := nocache.New() + if err != nil { + t.Errorf("error creating nocache: %v", err) + } + + db.Close() +} + +func TestSet(t *testing.T) { + db, err := nocache.New() + if err != nil { + t.Errorf("error creating nocache: %v", err) + } + + defer db.Close() + + err = db.Set("testkey", "testvalue") + if err != nil { + t.Errorf("error setting key-value pair: %v", err) + } +} + +func TestSetTTL(t *testing.T) { + db, err := nocache.New() + if err != nil { + t.Errorf("error creating nocache: %v", err) + } + + defer db.Close() + + err = db.Set("testkey", "testvalue", 1) + if err != nil { + t.Errorf("error setting key-value pair with TTL: %v", err) + } +} + +func TestGet(t *testing.T) { + db, err := nocache.New() + if err != nil { + t.Errorf("error creating nocache: %v", err) + } + + defer db.Close() + + err = db.Set("testkey", "testvalue") + if err != nil { + t.Errorf("error setting key-value pair: %v", err) + } + + var value string = "testvalue" + err = db.Get("testkey", &value) + if err != nil { + t.Errorf("error getting value: %v", err) + } + + if value != "testvalue" { + t.Errorf("expected value: testvalue, got: %v", value) + } +} + +func TestGetTTL(t *testing.T) { + db, err := nocache.New() + if err != nil { + t.Errorf("error creating nocache: %v", err) + } + + defer db.Close() + + err = db.Set("testkey", "testvalue", 1) + if err != nil { + t.Errorf("error setting key-value pair with TTL: %v", err) + } + + ttl, err := db.GetTTL("testkey") + if err != nil { + t.Errorf("error getting TTL: %v", err) + } + + if ttl != 0 { + t.Errorf("expected TTL: 0, got: %v", ttl) + } +} diff --git a/src/cache/pebble/pebble.go b/src/cache/pebble/pebble.go deleted file mode 100644 index 0229b42d..00000000 --- a/src/cache/pebble/pebble.go +++ /dev/null @@ -1,79 +0,0 @@ -package pebble - -import ( - "fmt" - "path" - "time" - - "github.com/cockroachdb/pebble" - "github.com/fxamacker/cbor/v2" - "github.com/hearchco/hearchco/src/anonymize" - "github.com/hearchco/hearchco/src/cache" - "github.com/rs/zerolog/log" -) - -type DB struct { - pdb *pebble.DB -} - -func New(dataDirPath string) *DB { - pebblePath := path.Join(dataDirPath, "database") - pdb, err := pebble.Open(pebblePath, &pebble.Options{}) - - if err != nil { - log.Error(). - Err(err). - Str("path", pebblePath). - Msg("pebble.New(): error opening pebble") - } else { - log.Info(). - Str("path", pebblePath). - Msg("Successfully opened pebble") - } - - return &DB{pdb: pdb} -} - -func (db *DB) Close() { - if err := db.pdb.Close(); err != nil { - log.Error().Err(err).Msg("pebble.Close(): error closing pebble") - } else { - log.Debug().Msg("Successfully closed pebble") - } -} - -func (db *DB) Set(k string, v cache.Value) error { - log.Debug().Msg("Caching...") - cacheTimer := time.Now() - - if val, err := cbor.Marshal(v); err != nil { - return fmt.Errorf("pebble.Set(): error marshaling value: %w", err) - } else if err := db.pdb.Set([]byte(anonymize.HashToSHA256B64(k)), val, pebble.NoSync); err != nil { - return fmt.Errorf("pebble.Set(): error setting KV to pebble: %w", err) - } else { - cacheTimeSince := time.Since(cacheTimer) - log.Trace(). - Int64("ms", cacheTimeSince.Milliseconds()). - Int64("ns", cacheTimeSince.Nanoseconds()). - Msg("Cached results") - } - return nil -} - -func (db *DB) Get(k string, o cache.Value) error { - v, c, err := db.pdb.Get([]byte(anonymize.HashToSHA256B64(k))) - val := []byte(v) // copy data before closing, casting needed for unmarshal - - if err == pebble.ErrNotFound { - log.Trace(). - Str("key", k). - Msg("Found no value in pebble") - } else if err != nil { - return fmt.Errorf("pebble.Get(): error getting value from pebble for key %v: %w", k, err) - } else if err := c.Close(); err != nil { - return fmt.Errorf("pebble.Get(): error closing io to pebble for key %v: %w", k, err) - } else if err := cbor.Unmarshal(val, o); err != nil { - return fmt.Errorf("pebble.Get(): failed unmarshaling value from pebble for key %v: %w", k, err) - } - return nil -} diff --git a/src/cache/redis/redis.go b/src/cache/redis/redis.go index a2ee0612..b02d9bec 100644 --- a/src/cache/redis/redis.go +++ b/src/cache/redis/redis.go @@ -18,7 +18,7 @@ type DB struct { ctx context.Context } -func New(ctx context.Context, config config.Redis) *DB { +func New(ctx context.Context, config config.Redis) (*DB, error) { rdb := redis.NewClient(&redis.Options{ Addr: fmt.Sprintf("%v:%v", config.Host, config.Port), Password: config.Password, @@ -36,7 +36,7 @@ func New(ctx context.Context, config config.Redis) *DB { Msg("Successful connection to redis") } - return &DB{rdb: rdb, ctx: ctx} + return &DB{rdb: rdb, ctx: ctx}, nil } func (db *DB) Close() { @@ -47,13 +47,18 @@ func (db *DB) Close() { } } -func (db *DB) Set(k string, v cache.Value) error { +func (db *DB) Set(k string, v cache.Value, ttl ...time.Duration) error { log.Debug().Msg("Caching...") cacheTimer := time.Now() + var setTtl time.Duration = 0 + if len(ttl) > 0 { + setTtl = ttl[0] + } + if val, err := cbor.Marshal(v); err != nil { return fmt.Errorf("redis.Set(): error marshaling value: %w", err) - } else if err := db.rdb.Set(db.ctx, anonymize.HashToSHA256B64(k), val, 0).Err(); err != nil { + } else if err := db.rdb.Set(db.ctx, anonymize.HashToSHA256B64(k), val, setTtl).Err(); err != nil { return fmt.Errorf("redis.Set(): error setting KV to redis: %w", err) } else { cacheTimeSince := time.Since(cacheTimer) @@ -62,21 +67,60 @@ func (db *DB) Set(k string, v cache.Value) error { Int64("ns", cacheTimeSince.Nanoseconds()). Msg("Cached results") } + return nil } -func (db *DB) Get(k string, o cache.Value) error { - v, err := db.rdb.Get(db.ctx, anonymize.HashToSHA256B64(k)).Result() - val := []byte(v) // copy data before closing, casting needed for unmarshal +func (db *DB) Get(k string, o cache.Value, hashed ...bool) error { + var kInput string + if len(hashed) > 0 && hashed[0] { + kInput = k + } else { + kInput = anonymize.HashToSHA256B64(k) + } + val, err := db.rdb.Get(db.ctx, kInput).Result() if err == redis.Nil { log.Trace(). - Str("key", k). + Str("key", kInput). Msg("Found no value in redis") } else if err != nil { - return fmt.Errorf("redis.Get(): error getting value from redis for key %v: %w", k, err) - } else if err := cbor.Unmarshal(val, o); err != nil { - return fmt.Errorf("redis.Get(): failed unmarshaling value from redis for key %v: %w", k, err) + return fmt.Errorf("redis.Get(): error getting value from redis for key %v: %w", kInput, err) + } else if err := cbor.Unmarshal([]byte(val), o); err != nil { + return fmt.Errorf("redis.Get(): failed unmarshaling value from redis for key %v: %w", kInput, err) } + return nil } + +// returns time until the key expires, not the time it will be considered expired +func (db *DB) GetTTL(k string, hashed ...bool) (time.Duration, error) { + var kInput string + if len(hashed) > 0 && hashed[0] { + kInput = k + } else { + kInput = anonymize.HashToSHA256B64(k) + } + + // returns time with time.Second precision + expiresIn, err := db.rdb.TTL(db.ctx, kInput).Result() + if err == redis.Nil { + log.Trace(). + Str("key", kInput). + Msg("Found no value in redis") + } else if err != nil { + return expiresIn, fmt.Errorf("redis.Get(): error getting value from redis for key %v: %w", kInput, err) + } + + /* + In Redis 2.6 or older the command returns -1 if the key does not exist or if the key exist but has no associated expire. + Starting with Redis 2.8 the return value in case of error changed: + The command returns -2 if the key does not exist. + The command returns -1 if the key exists but has no associated expire. + */ + if expiresIn < 0 { + expiresIn = 0 + } + + return expiresIn, nil +} diff --git a/src/cache/redis/redis_test.go b/src/cache/redis/redis_test.go new file mode 100644 index 00000000..9a90f8b8 --- /dev/null +++ b/src/cache/redis/redis_test.go @@ -0,0 +1,177 @@ +package redis_test + +import ( + "context" + "os" + "strconv" + "testing" + "time" + + "github.com/hearchco/hearchco/src/cache/redis" + "github.com/hearchco/hearchco/src/config" + redisog "github.com/redis/go-redis/v9" +) + +func newRedisConf() config.Redis { + redisHost := os.Getenv("REDIS_HOST") + if redisHost == "" { + redisHost = "localhost" + } + + var redisPort uint16 + redisPortStr := os.Getenv("REDIS_PORT") + + if redisPortStr == "" { + redisPort = 6379 + } else { + redisPortInt, err := strconv.Atoi(redisPortStr) + if err != nil { + panic(err) + } + redisPort = uint16(redisPortInt) + } + + redisPassword := os.Getenv("REDIS_PASSWORD") + + var redisDatabase uint8 + redisDatabaseStr := os.Getenv("REDIS_DATABASE") + if redisDatabaseStr == "" { + redisDatabase = 0 + } else { + redisDatabaseInt, err := strconv.Atoi(redisDatabaseStr) + if err != nil { + panic(err) + } + redisDatabase = uint8(redisDatabaseInt) + } + + return config.Redis{ + Host: redisHost, + Port: redisPort, + Password: redisPassword, + Database: redisDatabase, + } +} + +var redisConf = newRedisConf() + +func TestNew(t *testing.T) { + ctx := context.Background() + _, err := redis.New(ctx, redisConf) + if err != nil { + t.Errorf("error creating redis: %v", err) + } +} + +func TestClose(t *testing.T) { + ctx := context.Background() + db, err := redis.New(ctx, redisConf) + if err != nil { + t.Errorf("error creating redis: %v", err) + } + + db.Close() +} + +func TestSet(t *testing.T) { + ctx := context.Background() + db, err := redis.New(ctx, redisConf) + if err != nil { + t.Errorf("error creating redis: %v", err) + } + + defer db.Close() + + err = db.Set("testkeyset", "testvalue") + if err != nil { + t.Errorf("error setting key-value pair: %v", err) + } +} + +func TestSetTTL(t *testing.T) { + ctx := context.Background() + db, err := redis.New(ctx, redisConf) + if err != nil { + t.Errorf("error creating redis: %v", err) + } + + defer db.Close() + + err = db.Set("testkeysetttl", "testvalue", 100*time.Second) + if err != nil { + t.Errorf("error setting key-value pair with TTL: %v", err) + } +} + +func TestGet(t *testing.T) { + ctx := context.Background() + db, err := redis.New(ctx, redisConf) + if err != nil { + t.Errorf("error creating redis: %v", err) + } + + defer db.Close() + + err = db.Set("testkeyget", "testvalue") + if err != nil { + t.Errorf("error setting key-value pair: %v", err) + } + + var value string + err = db.Get("testkeyget", &value) + if err != nil { + t.Errorf("error getting value: %v", err) + } + + if value != "testvalue" { + t.Errorf("expected value: testvalue, got: %v", value) + } +} + +func TestGetTTL(t *testing.T) { + ctx := context.Background() + db, err := redis.New(ctx, redisConf) + if err != nil { + t.Errorf("error creating redis: %v", err) + } + + defer db.Close() + + err = db.Set("testkeygetttl", "testvalue", 100*time.Second) + if err != nil { + t.Errorf("error setting key-value pair with TTL: %v", err) + } + + ttl, err := db.GetTTL("testkeygetttl") + if err != nil { + t.Errorf("error getting TTL: %v", err) + } + + // TTL is not exact, so we check for a range + if ttl > 100*time.Second || ttl < 99*time.Second { + t.Errorf("expected 100s >= ttl >= 99s, got: %v", ttl) + } +} + +func TestGetExpired(t *testing.T) { + ctx := context.Background() + db, err := redis.New(ctx, redisConf) + if err != nil { + t.Errorf("error creating redis: %v", err) + } + + defer db.Close() + + err = db.Set("testkeygetexpired", "testvalue", 1*time.Second) + if err != nil { + t.Errorf("error setting key-value pair with TTL: %v", err) + } + + time.Sleep(1 * time.Second) + + var value string + err = db.Get("testkeygetexpired", &value) + if err != nil && err != redisog.Nil { + t.Errorf("error getting value: %v", err) + } +} diff --git a/src/cli/climode.go b/src/cli/climode.go index 24b3c8f6..0d9a4572 100644 --- a/src/cli/climode.go +++ b/src/cli/climode.go @@ -48,45 +48,9 @@ func Run(flags Flags, db cache.DB, conf *config.Config) { start := time.Now() - // todo: ctx cancelling (important since pebble is NoSync) - var results []result.Result - var foundInDB bool - gerr := db.Get(flags.Query, &results) - if gerr != nil { - // Error in reading cache is not returned, just logged - log.Error(). - Err(gerr). - Str("queryAnon", anonymize.String(flags.Query)). - Str("queryHash", anonymize.HashToSHA256B64(flags.Query)). - Msg("cli.Run(): failed accessing cache") - } else if results != nil { - foundInDB = true - } else { - foundInDB = false - } - - if foundInDB { - log.Debug(). - Str("queryAnon", anonymize.String(flags.Query)). - Str("queryHash", anonymize.HashToSHA256B64(flags.Query)). - Msg("Found results in cache") - } else { - log.Debug().Msg("Nothing found in cache, doing a clean search") - - results = search.PerformSearch(flags.Query, options, conf) - - serr := db.Set(flags.Query, results) - if serr != nil { - log.Error(). - Err(serr). - Str("queryAnon", anonymize.String(flags.Query)). - Str("queryHash", anonymize.HashToSHA256B64(flags.Query)). - Msg("cli.Run(): error updating database with search results") - } - } + results, foundInDB := search.Search(flags.Query, options, conf, db) duration := time.Since(start) - if !flags.Silent { printResults(results) } @@ -94,4 +58,6 @@ func Run(flags Flags, db cache.DB, conf *config.Config) { Int("number", len(results)). Int64("ms", duration.Milliseconds()). Msg("Found results") + + search.CacheAndUpdateResults(flags.Query, options, conf, db, results, foundInDB) } diff --git a/src/config/defaults.go b/src/config/defaults.go index 2e013cfc..56a10612 100644 --- a/src/config/defaults.go +++ b/src/config/defaults.go @@ -6,6 +6,7 @@ import ( "github.com/hearchco/hearchco/src/category" "github.com/hearchco/hearchco/src/engines" + "github.com/hearchco/hearchco/src/moretime" ) const DefaultLocale string = "en_US" @@ -133,7 +134,14 @@ func New() *Config { Port: 3030, FrontendUrl: "http://localhost:8000", Cache: Cache{ - Type: "pebble", + Type: "badger", + TTL: TTL{ + Time: moretime.Week, + RefreshTime: 3 * moretime.Day, + }, + Badger: Badger{ + Persist: true, + }, Redis: Redis{ Host: "localhost", Port: 6379, diff --git a/src/config/load.go b/src/config/load.go index 50c61610..f82d6bbe 100644 --- a/src/config/load.go +++ b/src/config/load.go @@ -8,6 +8,7 @@ import ( "github.com/hearchco/hearchco/src/category" "github.com/hearchco/hearchco/src/engines" + "github.com/hearchco/hearchco/src/moretime" "github.com/knadh/koanf/parsers/yaml" "github.com/knadh/koanf/providers/env" "github.com/knadh/koanf/providers/file" @@ -21,7 +22,19 @@ var LogDumpLocation string = "dump/" func (c *Config) fromReader(rc *ReaderConfig) { nc := Config{ - Server: rc.Server, + Server: Server{ + Port: rc.Server.Port, + FrontendUrl: rc.Server.FrontendUrl, + Cache: Cache{ + Type: rc.Server.Cache.Type, + TTL: TTL{ + Time: moretime.ConvertFancyTime(rc.Server.Cache.TTL.Time), + RefreshTime: moretime.ConvertFancyTime(rc.Server.Cache.TTL.RefreshTime), + }, + Badger: rc.Server.Cache.Badger, + Redis: rc.Server.Cache.Redis, + }, + }, Settings: map[engines.Name]Settings{}, Categories: map[category.Name]Category{}, } @@ -71,13 +84,21 @@ func (c *Config) fromReader(rc *ReaderConfig) { func (c *Config) getReader() ReaderConfig { rc := ReaderConfig{ - Server: c.Server, - Settings: map[string]Settings{}, + Server: ReaderServer{ + Port: c.Server.Port, + FrontendUrl: c.Server.FrontendUrl, + Cache: ReaderCache{ + Type: c.Server.Cache.Type, + TTL: ReaderTTL{ + Time: moretime.ConvertToFancyTime(c.Server.Cache.TTL.Time), + RefreshTime: moretime.ConvertToFancyTime(c.Server.Cache.TTL.RefreshTime), + }, + Badger: c.Server.Cache.Badger, + Redis: c.Server.Cache.Redis, + }, + }, RCategories: map[category.Name]ReaderCategory{}, - } - - for key, val := range c.Settings { - rc.Settings[key.ToLower()] = val + Settings: map[string]Settings{}, } for key, val := range c.Categories { @@ -99,6 +120,10 @@ func (c *Config) getReader() ReaderConfig { } } + for key, val := range c.Settings { + rc.Settings[key.ToLower()] = val + } + return rc } diff --git a/src/config/structs.go b/src/config/structs.go index 7330386a..e02cabf7 100644 --- a/src/config/structs.go +++ b/src/config/structs.go @@ -30,6 +30,31 @@ type Settings struct { Shortcut string `koanf:"shortcut"` } +// ReaderTTL is format in which the config is read from the config file +// in format +// example: 1s, 1m, 1h, 1d, 1w, 1M, 1y +// if unit is not specified, it is assumed to be milliseconds +type ReaderTTL struct { + // how long to store the results in cache + // setting this to 0 caches the results forever + // to disable caching set conf.Cache.Type to "none" + Time string `koanf:"time"` + // if the remaining TTL when retrieving from cache is less than this, update the cache entry and reset the TTL + // setting this to 0 disables this feature + // setting this to the same value (or higher) as Results will update the cache entry every time + RefreshTime string `koanf:"refreshtime"` +} +type TTL struct { + Time time.Duration + RefreshTime time.Duration +} + +type Badger struct { + // setting this to false will result in badger not persisting the cache to disk + // that means that badger will run in memory only + Persist bool `koanf:"persist"` +} + type Redis struct { Host string `koanf:"host"` Port uint16 `koanf:"port"` @@ -37,21 +62,45 @@ type Redis struct { Database uint8 `koanf:"database"` } +// ReaderCache is format in which the config is read from the config file +type ReaderCache struct { + // can be "none", "badger" or "redis" + Type string `koanf:"type"` + // has no effect if Type is "none" + TTL ReaderTTL `koanf:"ttl"` + // badger specific settings + Badger Badger `koanf:"badger"` + // redis specific settings + Redis Redis `koanf:"redis"` +} type Cache struct { - Type string `koanf:"type"` - Redis Redis `koanf:"redis"` + Type string + TTL TTL + Badger Badger + Redis Redis } -type Server struct { - Port int `koanf:"port"` +// ReaderServer is format in which the config is read from the config file +type ReaderServer struct { + // port on which the API server listens + Port int `koanf:"port"` + // frontend url needed for CORS FrontendUrl string `koanf:"frontendurl"` - Cache Cache `koanf:"cache"` + // cache settings + Cache ReaderCache `koanf:"cache"` +} +type Server struct { + Port int `koanf:"port"` + FrontendUrl string + Cache Cache } +// ReaderEngine is format in which the config is read from the config file type ReaderEngine struct { Enabled bool `koanf:"enabled"` } +// ReaderTimings is format in which the config is read from the config file // in miliseconds type ReaderTimings struct { // HardTimeout uint `koanf:"hardTimeout"` @@ -73,24 +122,24 @@ type Timings struct { Parallelism int } +// ReaderCategory is format in which the config is read from the config file type ReaderCategory struct { REngines map[string]ReaderEngine `koanf:"engines"` Ranking Ranking `koanf:"ranking"` RTimings ReaderTimings `koanf:"timings"` } - -type ReaderConfig struct { - Server Server `koanf:"server"` - RCategories map[category.Name]ReaderCategory `koanf:"categories"` - Settings map[string]Settings `koanf:"settings"` -} - type Category struct { Engines []engines.Name Ranking Ranking Timings Timings } +// ReaderConfig is format in which the config is read from the config file +type ReaderConfig struct { + Server ReaderServer `koanf:"server"` + RCategories map[category.Name]ReaderCategory `koanf:"categories"` + Settings map[string]Settings `koanf:"settings"` +} type Config struct { Server Server Categories map[category.Name]Category diff --git a/src/main.go b/src/main.go index 8acd4393..3da2893c 100644 --- a/src/main.go +++ b/src/main.go @@ -8,8 +8,8 @@ import ( "time" "github.com/hearchco/hearchco/src/cache" + "github.com/hearchco/hearchco/src/cache/badger" "github.com/hearchco/hearchco/src/cache/nocache" - "github.com/hearchco/hearchco/src/cache/pebble" "github.com/hearchco/hearchco/src/cache/redis" "github.com/hearchco/hearchco/src/cli" "github.com/hearchco/hearchco/src/config" @@ -49,13 +49,32 @@ func main() { // setup cache var db cache.DB + var err error switch conf.Server.Cache.Type { - case "pebble": - db = pebble.New(cliFlags.DataDirPath) + case "badger": + db, err = badger.New(cliFlags.DataDirPath, conf.Server.Cache.Badger) + if err != nil { + log.Fatal(). + Err(err). + Msg("main.main(): failed creating a badger cache") + // ^FATAL + } case "redis": - db = redis.New(ctx, conf.Server.Cache.Redis) + db, err = redis.New(ctx, conf.Server.Cache.Redis) + if err != nil { + log.Fatal(). + Err(err). + Msg("main.main(): failed creating a redis cache") + // ^FATAL + } default: - db = nocache.New() + db, err = nocache.New() + if err != nil { + log.Fatal(). + Err(err). + Msg("main.main(): failed creating a nocache") + // ^FATAL + } log.Warn().Msg("Running without caching!") } diff --git a/src/moretime/fancy.go b/src/moretime/fancy.go new file mode 100644 index 00000000..a9577167 --- /dev/null +++ b/src/moretime/fancy.go @@ -0,0 +1,48 @@ +package moretime + +import ( + "strconv" + "time" + + "github.com/rs/zerolog/log" +) + +func handleAtoi(s string) int64 { + i, err := strconv.Atoi(s) + if err != nil { + log.Panic(). + Err(err). + Msg("failed converting string to int") + // ^PANIC + } + return int64(i) +} + +func convertToDurationWithoutLastChar(s string) time.Duration { + return time.Duration(handleAtoi(s[:len(s)-1])) +} + +func ConvertFancyTime(fancy string) time.Duration { + switch fancy[len(fancy)-1] { + case 'y': + return convertToDurationWithoutLastChar(fancy) * Year + case 'M': + return convertToDurationWithoutLastChar(fancy) * Month + case 'w': + return convertToDurationWithoutLastChar(fancy) * Week + case 'd': + return convertToDurationWithoutLastChar(fancy) * Day + case 'h': + return convertToDurationWithoutLastChar(fancy) * time.Hour + case 'm': + return convertToDurationWithoutLastChar(fancy) * time.Minute + case 's': + return convertToDurationWithoutLastChar(fancy) * time.Second + default: + return time.Duration(handleAtoi(fancy)) * time.Millisecond + } +} + +func ConvertToFancyTime(d time.Duration) string { + return strconv.Itoa(int(d.Milliseconds())) +} diff --git a/src/moretime/time.go b/src/moretime/time.go new file mode 100644 index 00000000..8b0ffd5c --- /dev/null +++ b/src/moretime/time.go @@ -0,0 +1,10 @@ +package moretime + +import "time" + +const Day = 24 * time.Hour +const Week = 7 * Day +const Month = 30 * Day +const Quarter = 3 * Month +const HalfYear = 6 * Month +const Year = 365 * Day diff --git a/src/router/search.go b/src/router/search.go index 480ce612..ac9ad080 100644 --- a/src/router/search.go +++ b/src/router/search.go @@ -7,9 +7,7 @@ import ( "github.com/gin-gonic/gin" "github.com/goccy/go-json" - "github.com/rs/zerolog/log" - "github.com/hearchco/hearchco/src/anonymize" "github.com/hearchco/hearchco/src/bucket/result" "github.com/hearchco/hearchco/src/cache" "github.com/hearchco/hearchco/src/category" @@ -95,32 +93,7 @@ func Search(c *gin.Context, conf *config.Config, db cache.DB) error { Mobile: isMobile, } - var results []result.Result - var foundInDB bool - gerr := db.Get(query, &results) - if gerr != nil { - // Error in reading cache is not returned, just logged - log.Error(). - Err(gerr). - Str("queryAnon", anonymize.String(query)). - Str("queryHash", anonymize.HashToSHA256B64(query)). - Msg("router.Search(): failed accessing cache") - } else if results != nil { - foundInDB = true - } else { - foundInDB = false - } - - if foundInDB { - log.Debug(). - Str("queryAnon", anonymize.String(query)). - Str("queryHash", anonymize.HashToSHA256B64(query)). - Msg("Found results in cache") - } else { - log.Debug().Msg("Nothing found in cache, doing a clean search") - - results = search.PerformSearch(query, options, conf) - } + results, foundInDB := search.Search(query, options, conf, db) resultsShort := result.Shorten(results) if resultsJson, err := json.Marshal(resultsShort); err != nil { @@ -130,17 +103,7 @@ func Search(c *gin.Context, conf *config.Config, db cache.DB) error { c.String(http.StatusOK, string(resultsJson)) } - if !foundInDB { - serr := db.Set(query, results) - if serr != nil { - // Error in updating cache is not returned, just logged - log.Error(). - Err(serr). - Str("queryAnon", anonymize.String(query)). - Str("queryHash", anonymize.HashToSHA256B64(query)). - Msg("router.Search(): error updating database with search results") - } - } + search.CacheAndUpdateResults(query, options, conf, db, results, foundInDB) } return nil } diff --git a/src/search/cache.go b/src/search/cache.go new file mode 100644 index 00000000..ed393c17 --- /dev/null +++ b/src/search/cache.go @@ -0,0 +1,55 @@ +package search + +import ( + "github.com/hearchco/hearchco/src/anonymize" + "github.com/hearchco/hearchco/src/bucket/result" + "github.com/hearchco/hearchco/src/cache" + "github.com/hearchco/hearchco/src/config" + "github.com/hearchco/hearchco/src/engines" + "github.com/rs/zerolog/log" +) + +func CacheAndUpdateResults(query string, options engines.Options, conf *config.Config, db cache.DB, results []result.Result, foundInDB bool) { + if !foundInDB { + log.Debug(). + Str("queryAnon", anonymize.String(query)). + Str("queryHash", anonymize.HashToSHA256B64(query)). + Msg("Caching results...") + serr := db.Set(query, results, conf.Server.Cache.TTL.Time) + if serr != nil { + log.Error(). + Err(serr). + Str("queryAnon", anonymize.String(query)). + Str("queryHash", anonymize.HashToSHA256B64(query)). + Msg("cli.Run(): error updating database with search results") + } + } else { + log.Debug(). + Str("queryAnon", anonymize.String(query)). + Str("queryHash", anonymize.HashToSHA256B64(query)). + Msg("Checking if results need to be updated") + ttl, terr := db.GetTTL(query) + if terr != nil { + log.Error(). + Err(terr). + Str("queryAnon", anonymize.String(query)). + Str("queryHash", anonymize.HashToSHA256B64(query)). + Msg("cli.Run(): error getting TTL from database") + } else if ttl < conf.Server.Cache.TTL.RefreshTime { + log.Info(). + Str("queryAnon", anonymize.String(query)). + Str("queryHash", anonymize.HashToSHA256B64(query)). + Msg("Updating results...") + newResults := PerformSearch(query, options, conf) + uerr := db.Set(query, newResults, conf.Server.Cache.TTL.Time) + if uerr != nil { + // Error in updating cache is not returned, just logged + log.Error(). + Err(uerr). + Str("queryAnon", anonymize.String(query)). + Str("queryHash", anonymize.HashToSHA256B64(query)). + Msg("cli.Run(): error replacing old results while updating database") + } + } + } +} diff --git a/src/search/perform.go b/src/search/perform.go new file mode 100644 index 00000000..26478c68 --- /dev/null +++ b/src/search/perform.go @@ -0,0 +1,139 @@ +package search + +import ( + "context" + "fmt" + "net/url" + "strings" + "time" + + "github.com/hearchco/hearchco/src/anonymize" + "github.com/hearchco/hearchco/src/bucket" + "github.com/hearchco/hearchco/src/bucket/result" + "github.com/hearchco/hearchco/src/category" + "github.com/hearchco/hearchco/src/config" + "github.com/hearchco/hearchco/src/engines" + "github.com/hearchco/hearchco/src/rank" + "github.com/rs/zerolog/log" + "github.com/sourcegraph/conc" +) + +func PerformSearch(query string, options engines.Options, conf *config.Config) []result.Result { + searchTimer := time.Now() + + relay := bucket.Relay{ + ResultMap: make(map[string]*result.Result), + } + + query, timings, toRun := procBang(query, &options, conf) + + query = url.QueryEscape(query) + log.Debug(). + Str("queryAnon", anonymize.String(query)). + Str("queryHash", anonymize.HashToSHA256B64(query)). + Msg("Searching") + + resTimer := time.Now() + log.Debug().Msg("Waiting for results from engines...") + var worker conc.WaitGroup + runEngines(toRun, timings, conf.Settings, query, &worker, &relay, options) + worker.Wait() + log.Debug(). + Int64("ms", time.Since(resTimer).Milliseconds()). + Msg("Got results") + + rankTimer := time.Now() + log.Debug().Msg("Ranking...") + results := rank.Rank(relay.ResultMap, conf.Categories[options.Category].Ranking) // have to make copy, since its a map value + rankTimeSince := time.Since(rankTimer) + log.Debug(). + Int64("ms", rankTimeSince.Milliseconds()). + Int64("ns", rankTimeSince.Nanoseconds()). + Msg("Finished ranking") + + log.Debug(). + Int64("ms", time.Since(searchTimer).Milliseconds()). + Msg("Found results") + + return results +} + +// engine_searcher, NewEngineStarter() use this. +type EngineSearch func(context.Context, string, *bucket.Relay, engines.Options, config.Settings, config.Timings) error + +func runEngines(engs []engines.Name, timings config.Timings, settings map[engines.Name]config.Settings, query string, worker *conc.WaitGroup, relay *bucket.Relay, options engines.Options) { + config.EnabledEngines = engs + log.Info(). + Int("number", len(config.EnabledEngines)). + Str("engines", fmt.Sprintf("%v", config.EnabledEngines)). + Msg("Enabled engines") + + engineStarter := NewEngineStarter() + for i := range engs { + eng := engs[i] // dont change for to `for _, eng := range engs {`, eng retains the same address throughout the whole loop + worker.Go(func() { + // if an error can be handled inside, it wont be returned + // runs the Search function in the engine package + err := engineStarter[eng](context.Background(), query, relay, options, settings[eng], timings) + if err != nil { + log.Error(). + Err(err). + Str("engine", eng.String()). + Msg("search.runEngines(): error while searching") + } + }) + } +} + +func procBang(query string, options *engines.Options, conf *config.Config) (string, config.Timings, []engines.Name) { + useSpec, specEng := procSpecificEngine(query, options, conf) + goodCat := procCategory(query, options) + if !goodCat && !useSpec && query[0] == '!' { + // options.category is set to GENERAL + log.Debug(). + Str("queryAnon", anonymize.String(query)). + Str("queryHash", anonymize.HashToSHA256B64(query)). + Msg("search.procBang(): invalid bang (not category or engine shortcut)") + } + + query = trimBang(query) + + if useSpec { + return query, conf.Categories[category.GENERAL].Timings, []engines.Name{specEng} + } else { + return query, conf.Categories[options.Category].Timings, conf.Categories[options.Category].Engines + } +} + +func trimBang(query string) string { + if (query)[0] == '!' { + return strings.SplitN(query, " ", 2)[1] + } + return query +} + +func procSpecificEngine(query string, options *engines.Options, conf *config.Config) (bool, engines.Name) { + if query[0] != '!' { + return false, engines.UNDEFINED + } + sp := strings.SplitN(query, " ", 2) + bangWord := sp[0][1:] + for key, val := range conf.Settings { + if strings.EqualFold(bangWord, val.Shortcut) || strings.EqualFold(bangWord, key.String()) { + return true, key + } + } + + return false, engines.UNDEFINED +} + +func procCategory(query string, options *engines.Options) bool { + cat := category.FromQuery(query) + if cat != "" { + options.Category = cat + } + if options.Category == "" { + options.Category = category.GENERAL + } + return cat != "" +} diff --git a/src/search/search.go b/src/search/search.go index 26478c68..2bf519aa 100644 --- a/src/search/search.go +++ b/src/search/search.go @@ -1,139 +1,43 @@ package search import ( - "context" - "fmt" - "net/url" - "strings" - "time" - "github.com/hearchco/hearchco/src/anonymize" - "github.com/hearchco/hearchco/src/bucket" "github.com/hearchco/hearchco/src/bucket/result" - "github.com/hearchco/hearchco/src/category" + "github.com/hearchco/hearchco/src/cache" "github.com/hearchco/hearchco/src/config" "github.com/hearchco/hearchco/src/engines" - "github.com/hearchco/hearchco/src/rank" "github.com/rs/zerolog/log" - "github.com/sourcegraph/conc" ) -func PerformSearch(query string, options engines.Options, conf *config.Config) []result.Result { - searchTimer := time.Now() - - relay := bucket.Relay{ - ResultMap: make(map[string]*result.Result), - } - - query, timings, toRun := procBang(query, &options, conf) - - query = url.QueryEscape(query) - log.Debug(). - Str("queryAnon", anonymize.String(query)). - Str("queryHash", anonymize.HashToSHA256B64(query)). - Msg("Searching") - - resTimer := time.Now() - log.Debug().Msg("Waiting for results from engines...") - var worker conc.WaitGroup - runEngines(toRun, timings, conf.Settings, query, &worker, &relay, options) - worker.Wait() - log.Debug(). - Int64("ms", time.Since(resTimer).Milliseconds()). - Msg("Got results") - - rankTimer := time.Now() - log.Debug().Msg("Ranking...") - results := rank.Rank(relay.ResultMap, conf.Categories[options.Category].Ranking) // have to make copy, since its a map value - rankTimeSince := time.Since(rankTimer) - log.Debug(). - Int64("ms", rankTimeSince.Milliseconds()). - Int64("ns", rankTimeSince.Nanoseconds()). - Msg("Finished ranking") - - log.Debug(). - Int64("ms", time.Since(searchTimer).Milliseconds()). - Msg("Found results") - - return results -} - -// engine_searcher, NewEngineStarter() use this. -type EngineSearch func(context.Context, string, *bucket.Relay, engines.Options, config.Settings, config.Timings) error - -func runEngines(engs []engines.Name, timings config.Timings, settings map[engines.Name]config.Settings, query string, worker *conc.WaitGroup, relay *bucket.Relay, options engines.Options) { - config.EnabledEngines = engs - log.Info(). - Int("number", len(config.EnabledEngines)). - Str("engines", fmt.Sprintf("%v", config.EnabledEngines)). - Msg("Enabled engines") - - engineStarter := NewEngineStarter() - for i := range engs { - eng := engs[i] // dont change for to `for _, eng := range engs {`, eng retains the same address throughout the whole loop - worker.Go(func() { - // if an error can be handled inside, it wont be returned - // runs the Search function in the engine package - err := engineStarter[eng](context.Background(), query, relay, options, settings[eng], timings) - if err != nil { - log.Error(). - Err(err). - Str("engine", eng.String()). - Msg("search.runEngines(): error while searching") - } - }) +func Search(query string, options engines.Options, conf *config.Config, db cache.DB) ([]result.Result, bool) { + var results []result.Result + var foundInDB bool + gerr := db.Get(query, &results) + if gerr != nil { + // Error in reading cache is not returned, just logged + log.Error(). + Err(gerr). + Str("queryAnon", anonymize.String(query)). + Str("queryHash", anonymize.HashToSHA256B64(query)). + Msg("cli.Run(): failed accessing cache") + } else if results != nil { + foundInDB = true + } else { + foundInDB = false } -} -func procBang(query string, options *engines.Options, conf *config.Config) (string, config.Timings, []engines.Name) { - useSpec, specEng := procSpecificEngine(query, options, conf) - goodCat := procCategory(query, options) - if !goodCat && !useSpec && query[0] == '!' { - // options.category is set to GENERAL + if foundInDB { log.Debug(). Str("queryAnon", anonymize.String(query)). Str("queryHash", anonymize.HashToSHA256B64(query)). - Msg("search.procBang(): invalid bang (not category or engine shortcut)") - } - - query = trimBang(query) - - if useSpec { - return query, conf.Categories[category.GENERAL].Timings, []engines.Name{specEng} + Msg("Found results in cache") } else { - return query, conf.Categories[options.Category].Timings, conf.Categories[options.Category].Engines - } -} - -func trimBang(query string) string { - if (query)[0] == '!' { - return strings.SplitN(query, " ", 2)[1] - } - return query -} - -func procSpecificEngine(query string, options *engines.Options, conf *config.Config) (bool, engines.Name) { - if query[0] != '!' { - return false, engines.UNDEFINED - } - sp := strings.SplitN(query, " ", 2) - bangWord := sp[0][1:] - for key, val := range conf.Settings { - if strings.EqualFold(bangWord, val.Shortcut) || strings.EqualFold(bangWord, key.String()) { - return true, key - } + log.Debug(). + Str("queryAnon", anonymize.String(query)). + Str("queryHash", anonymize.HashToSHA256B64(query)). + Msg("Nothing found in cache, doing a clean search") + results = PerformSearch(query, options, conf) } - return false, engines.UNDEFINED -} - -func procCategory(query string, options *engines.Options) bool { - cat := category.FromQuery(query) - if cat != "" { - options.Category = cat - } - if options.Category == "" { - options.Category = category.GENERAL - } - return cat != "" + return results, foundInDB }