From 15e32ee2d27650da05cf583c072e49f387866f21 Mon Sep 17 00:00:00 2001 From: Giovanni Marigi <​gio.marigi@gmail.com> Date: Fri, 26 Jul 2024 20:57:36 +0200 Subject: [PATCH 01/51] support for schema registry with CSFLE e2e encryption --- go.mod | 44 ++++++- go.sum | 41 +++++- pkg/producers/kafka/kafkaProducer.go | 121 +++++++++++++++++- .../kafka/registry.properties.example | 7 +- pkg/types/payment_credit_card.avsc | 3 +- 5 files changed, 206 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index aff90e19..e0108436 100644 --- a/go.mod +++ b/go.mod @@ -24,25 +24,68 @@ require ( cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect cloud.google.com/go/compute/metadata v0.3.0 // indirect cloud.google.com/go/iam v1.1.8 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.1.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect + github.com/aws/aws-sdk-go-v2 v1.26.1 // indirect + github.com/aws/aws-sdk-go-v2/config v1.27.10 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.10 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect + github.com/aws/aws-sdk-go-v2/service/kms v1.30.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.20.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect + github.com/aws/smithy-go v1.20.2 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.1 // indirect + github.com/cenkalti/backoff/v3 v3.0.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-jose/go-jose/v3 v3.0.3 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang-jwt/jwt/v5 v5.2.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.5 // indirect + github.com/hamba/avro/v2 v2.20.1 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-retryablehttp v0.7.5 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 // indirect + github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect + github.com/hashicorp/go-sockaddr v1.0.2 // indirect + github.com/hashicorp/vault/api v1.12.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.7 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/montanaflynn/stats v0.7.1 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/ryanuber/go-glob v1.0.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect + github.com/tink-crypto/tink-go-gcpkms/v2 v2.1.0 // indirect + github.com/tink-crypto/tink-go-hcvault/v2 v2.1.0 // indirect + github.com/tink-crypto/tink-go/v2 v2.1.0 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect @@ -74,7 +117,6 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/heetch/avro v0.4.5 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/invopop/jsonschema v0.12.0 // indirect github.com/magiconair/properties v1.8.7 // indirect diff --git a/go.sum b/go.sum index d886ef3a..d36c6cea 100644 --- a/go.sum +++ b/go.sum @@ -46,6 +46,7 @@ github.com/actgardner/gogen-avro/v10 v10.2.1 h1:z3pOGblRjAJCYpkIJ8CmbMJdksi4rAha github.com/actgardner/gogen-avro/v10 v10.2.1/go.mod h1:QUhjeHPchheYmMDni/Nx7VB0RsT/ee8YIgGY/xpEQgQ= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.54.14 h1:llJ60MzLzovyDE/rEDbUjS1cICh7krk1PwQwNlKRoeQ= github.com/aws/aws-sdk-go v1.54.14/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA= @@ -80,6 +81,7 @@ github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPn github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -124,6 +126,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/buildx v0.14.0 h1:FxqcfE7xgeEC4oQlKLpuvfobRDVDXrHE3jByM+mdyqk= github.com/docker/buildx v0.14.0/go.mod h1:Vy/2lC9QsJvo33+7KKkN/GDE5WxnVqW0/dpcN7ZqPJY= github.com/docker/cli v26.1.0+incompatible h1:+nwRy8Ocd8cYNQ60mozDDICICD8aoFGtlPXifX/UQ3Y= @@ -156,6 +160,9 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -183,6 +190,8 @@ github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2Kv github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= +github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= @@ -221,8 +230,10 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= @@ -246,10 +257,17 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaW github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= +github.com/hamba/avro/v2 v2.20.1 h1:3WByQiVn7wT7d27WQq6pvBRC00FVOrniP6u67FLA/2E= +github.com/hamba/avro/v2 v2.20.1/go.mod h1:xHiKXbISpb3Ovc809XdzWow+XGTn+Oyf/F9aZbTLAig= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= +github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= @@ -258,6 +276,7 @@ github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5O github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 h1:om4Al8Oy7kCm/B86rLCLah4Dt5Aa0Fr5rYBG60OzwHQ= github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6/go.mod h1:QmrqtbKuxxSWTN3ETMPuB+VtEiBJ/A9XhoYGv8E1uD8= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.1/go.mod h1:gKOamz3EwoIoJq7mlMIRBpVTAUn8qPCrEclOKKWhD3U= github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= @@ -268,8 +287,6 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/vault/api v1.12.1 h1:WzGN4X5jrJdNO39g6Sa55djNio3I9DxEBOTmCZE7tm0= github.com/hashicorp/vault/api v1.12.1/go.mod h1:1pqP/sErScodde+ybJCyP+ONC4jzEg7Dmawg/QLWo1k= -github.com/heetch/avro v0.4.5 h1:BSnj4wEeUG1IjMTm9/tBwQnV3euuIVa1mRWHnm1t8VU= -github.com/heetch/avro v0.4.5/go.mod h1:gxf9GnbjTXmWmqxhdNbAMcZCjpye7RV5r9t3Q0dL6ws= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/in-toto/in-toto-golang v0.5.0 h1:hb8bgwr0M2hGdDsLjkJ3ZqJ8JFLL/tgYdAxF/XEFBbY= @@ -304,8 +321,10 @@ github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0V github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= @@ -318,10 +337,13 @@ github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1f github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU= github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= @@ -348,6 +370,7 @@ github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= @@ -375,6 +398,7 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= @@ -395,6 +419,7 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= @@ -442,6 +467,9 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -539,6 +567,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -559,6 +588,7 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -571,6 +601,7 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -579,12 +610,17 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -593,6 +629,7 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= diff --git a/pkg/producers/kafka/kafkaProducer.go b/pkg/producers/kafka/kafkaProducer.go index a934e226..5bccd242 100644 --- a/pkg/producers/kafka/kafkaProducer.go +++ b/pkg/producers/kafka/kafkaProducer.go @@ -27,12 +27,21 @@ import ( "fmt" "github.com/confluentinc/confluent-kafka-go/v2/kafka" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption/awskms" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption/azurekms" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption/gcpkms" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption/hcvault" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde" - "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde/avro" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde/avrov2" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde/jsonschema" "github.com/ugol/jr/pkg/types" + "io/ioutil" "log" "os" + "path/filepath" + "regexp" + "runtime" "strconv" "strings" "time" @@ -46,6 +55,7 @@ type KafkaManager struct { Topic string Serializer string TemplateType string + fleEnabled bool } func (k *KafkaManager) Initialize(configFile string) { @@ -75,6 +85,89 @@ func (k *KafkaManager) InitializeSchemaRegistry(configFile string) { log.Fatalf("Failed to create schema registry client: %s", err) } + if k.Serializer == "avro" || k.Serializer == "avro-generic" { + // verify if CSFLE must be enabled + if conf["kekName"] != "" && conf["kmsType"] != "" && conf["kmsKeyID"] != "" { + // register kms providers + awskms.Register() + azurekms.Register() + gcpkms.Register() + hcvault.Register() + encryption.Register() + + // load avro schema file: CSFLE requires schema registration + _, currentFilePath, _, _ := runtime.Caller(0) + currentDir := filepath.Dir(currentFilePath) + filePath := filepath.Join(currentDir, "../../types/"+k.TemplateType+".avsc") + file, err := os.Open(filePath) + if err != nil { + log.Fatalf("Failed to open file: %s", err) + } + defer file.Close() + + content, err := ioutil.ReadAll(file) + if err != nil { + log.Fatalf("Failed to read file: %s", err) + } + contentString := string(content) + + // check presence of PII in schema + substring := `"confluent:tags": [ "PII" ]` + normalizedJSON := normalizeWhitespace(contentString) + normalizedSubstring := normalizeWhitespace(substring) + + if strings.Contains(normalizedJSON, normalizedSubstring) { + // upper-casing the first letter of the fields --> name - required by https://pkg.go.dev/github.com/actgardner/gogen-avro#readme-naming + re := regexp.MustCompile(`"name"\s*:\s*"([^"]+)"`) + result := re.ReplaceAllStringFunc(contentString, func(match string) string { + // extract the name part after "name: " + parts := re.FindStringSubmatch(match) + fmt.Print(len(parts)) + if len(parts) > 1 { + name := parts[1] + // capitalize the first letter of the name + capitalized := capitalizeFirstLetter(name) + // replace the original match with the new capitalized version + return "\"name\":" + "\"" + capitalized + "\"" + } + return match + }) + + // register the avro schema adding rule set: PII + schema := schemaregistry.SchemaInfo{ + Schema: result, + SchemaType: "AVRO", + RuleSet: &schemaregistry.RuleSet{ + DomainRules: []schemaregistry.Rule{ + { + Name: "encryptPII", + Kind: "TRANSFORM", + Mode: "WRITEREAD", + Type: "ENCRYPT", + Tags: []string{"PII"}, + Params: map[string]string{ + "encrypt.kek.name": conf["kekName"], + "encrypt.kms.type": conf["kmsType"], + "encrypt.kms.key.id": conf["kmsKeyID"], + }, + OnFailure: "ERROR,NONE", + }, + }, + }, + } + _, err = k.schema.Register(k.Topic+"-value", schema, true) + if err != nil { + fmt.Printf("Failed to register schema: %s\n", err) + os.Exit(1) + } + + k.fleEnabled = true + + } + + } + } + k.schemaRegistry = true } @@ -93,10 +186,15 @@ func (k *KafkaManager) Produce(key []byte, data []byte, o any) { if k.schemaRegistry { var err error - if k.Serializer == "avro" { - ser, err = avro.NewSpecificSerializer(k.schema, serde.ValueSerde, avro.NewSerializerConfig()) - } else if k.Serializer == "avro-generic" { - ser, err = avro.NewGenericSerializer(k.schema, serde.ValueSerde, avro.NewSerializerConfig()) + + if k.Serializer == "avro" || k.Serializer == "avro-generic" { + serConfig := avrov2.NewSerializerConfig() + // CSFLE requires auto register to false + if k.fleEnabled { + serConfig.AutoRegisterSchemas = false + serConfig.UseLatestVersion = true + } + ser, err = avrov2.NewSerializer(k.schema, serde.ValueSerde, serConfig) } else if k.Serializer == "protobuf" { //ser, err = protobuf.NewSerializer(k.schema, serde.ValueSerde, protobuf.NewSerializerConfig()) log.Fatal("Protobuf not yet implemented") @@ -122,6 +220,7 @@ func (k *KafkaManager) Produce(key []byte, data []byte, o any) { } else { data = payload } + } } @@ -259,3 +358,15 @@ func convertInKafkaConfig(m map[string]string) kafka.ConfigMap { } return conf } + +func capitalizeFirstLetter(s string) string { + if len(s) == 0 { + return s + } + return strings.ToUpper(string(s[0])) + s[1:] +} + +func normalizeWhitespace(s string) string { + re := regexp.MustCompile(`\s+`) + return re.ReplaceAllString(s, " ") +} diff --git a/pkg/producers/kafka/registry.properties.example b/pkg/producers/kafka/registry.properties.example index a3555498..176733ea 100644 --- a/pkg/producers/kafka/registry.properties.example +++ b/pkg/producers/kafka/registry.properties.example @@ -2,4 +2,9 @@ schemaRegistryURL= schemaRegistryUser= -schemaRegistryPassword= \ No newline at end of file +schemaRegistryPassword= + +# Properties for Field Level Encryption (FLE) +kekName= +kmsType= +kmsKeyID= diff --git a/pkg/types/payment_credit_card.avsc b/pkg/types/payment_credit_card.avsc index 1debc958..6f9e382a 100644 --- a/pkg/types/payment_credit_card.avsc +++ b/pkg/types/payment_credit_card.avsc @@ -9,7 +9,8 @@ }, { "name": "card_number", - "type": "string" + "type": "string", + "confluent:tags": [ "PII" ] }, { "name": "cvv", From 237badfa3e9b5774664b68950593b733e9680514 Mon Sep 17 00:00:00 2001 From: eljeko Date: Thu, 1 Aug 2024 19:49:17 +0200 Subject: [PATCH 02/51] fixed http session for server mode --- go.mod | 2 ++ go.sum | 4 ++++ pkg/cmd/server.go | 54 +++++++++++++++++++++++++++++++++++------------ 3 files changed, 46 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index e0108436..195f685a 100644 --- a/go.mod +++ b/go.mod @@ -59,6 +59,8 @@ require ( github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.5 // indirect + github.com/gorilla/securecookie v1.1.2 // indirect + github.com/gorilla/sessions v1.3.0 // indirect github.com/hamba/avro/v2 v2.20.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect diff --git a/go.sum b/go.sum index d36c6cea..d2679108 100644 --- a/go.sum +++ b/go.sum @@ -251,6 +251,10 @@ github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBY github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/gorilla/securecookie v1.1.2 h1:YCIWL56dvtr73r6715mJs5ZvhtnY73hBvEF8kXD8ePA= +github.com/gorilla/securecookie v1.1.2/go.mod h1:NfCASbcHqRSY+3a8tlWJwsQap2VX5pwzwo4h3eOamfo= +github.com/gorilla/sessions v1.3.0 h1:XYlkq7KcpOB2ZhHBPv5WpjMIxrQosiZanfoy1HLZFzg= +github.com/gorilla/sessions v1.3.0/go.mod h1:ePLdVu+jbEgHH+KWw8I1z2wqd0BAdAQh/8LRvBeoNcQ= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= diff --git a/pkg/cmd/server.go b/pkg/cmd/server.go index 36d3ff56..3613aabf 100644 --- a/pkg/cmd/server.go +++ b/pkg/cmd/server.go @@ -2,23 +2,26 @@ package cmd import ( "bytes" + "context" _ "embed" "encoding/base64" "encoding/json" "fmt" + "log" + "net/http" + "path/filepath" + "strings" + "text/template" + "time" + "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" + "github.com/gorilla/sessions" "github.com/spf13/cobra" "github.com/ugol/jr/pkg/configuration" "github.com/ugol/jr/pkg/constants" "github.com/ugol/jr/pkg/emitter" "github.com/ugol/jr/pkg/functions" - "log" - "net/http" - "path/filepath" - "strings" - "text/template" - "time" ) //go:embed html/index.html @@ -63,12 +66,11 @@ var font_awesome_js string //go:embed html/images/jr_logo.png var jr_logo_png []byte -var lastTemplateSubmittedValue []byte -var lastTemplateSubmittedisJsonOutputValue []byte - var firstRun = make(map[string]bool) var emitterToRun = make(map[string][]emitter.Emitter) +var store = sessions.NewCookieStore([]byte("templates")) + var serverCmd = &cobra.Command{ Use: "server", Short: "Starts the jr http server", @@ -95,6 +97,7 @@ var serverCmd = &cobra.Command{ router.Use(middleware.Logger) router.Use(middleware.Recoverer) router.Use(middleware.Timeout(60 * time.Second)) + router.Use(SessionMiddleware) //comment for local dev embeddedFileRoutes(router) @@ -135,6 +138,14 @@ var serverCmd = &cobra.Command{ }, } +func SessionMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + session, _ := store.Get(r, "session-name") + r = r.WithContext(context.WithValue(r.Context(), "session", session)) + next.ServeHTTP(w, r) + }) +} + func embeddedFileRoutes(router chi.Router) { router.Get("/", func(w http.ResponseWriter, r *http.Request) { @@ -357,9 +368,18 @@ func loadLastStatus(w http.ResponseWriter, r *http.Request) { var response bytes.Buffer response.WriteString("{") - lastTemplateSubmittedValueB64 := base64.StdEncoding.EncodeToString(lastTemplateSubmittedValue) + + session := r.Context().Value("session").(*sessions.Session) + lastTemplateSubmittedValue_without_type, _ := session.Values["lastTemplateSubmittedValue"] + lastTemplateSubmittedValue, _ := lastTemplateSubmittedValue_without_type.(string) + + lastTemplateSubmittedisJsonOutputValue_without_type, _ := session.Values["lastTemplateSubmittedisJsonOutputValue"] + lastTemplateSubmittedisJsonOutputValue, _ := lastTemplateSubmittedisJsonOutputValue_without_type.(string) + + lastTemplateSubmittedValueB64 := base64.StdEncoding.EncodeToString([]byte(lastTemplateSubmittedValue)) + response.WriteString("\"template\": \"" + lastTemplateSubmittedValueB64 + "\",") - response.WriteString("\"isJsonOutput\": \"" + string(lastTemplateSubmittedisJsonOutputValue) + "\"") + response.WriteString("\"isJsonOutput\": \"" + lastTemplateSubmittedisJsonOutputValue + "\"") response.WriteString("}") _, err := w.Write(response.Bytes()) @@ -371,16 +391,22 @@ func loadLastStatus(w http.ResponseWriter, r *http.Request) { func executeTemplate(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "plain/text") + errorFormParse := r.ParseForm() if errorFormParse != nil { log.Println("errorFormParse ", errorFormParse) http.Error(w, errorFormParse.Error(), http.StatusInternalServerError) } - lastTemplateSubmittedValue = []byte(r.Form.Get("template")) - lastTemplateSubmittedisJsonOutputValue = []byte(r.Form.Get("isJsonOutput")) + var lastTemplateSubmittedValue = r.Form.Get("template") + var lastTemplateSubmittedisJsonOutputValue = r.Form.Get("isJsonOutput") + + session := r.Context().Value("session").(*sessions.Session) + session.Values["lastTemplateSubmittedValue"] = lastTemplateSubmittedValue + session.Values["lastTemplateSubmittedisJsonOutputValue"] = lastTemplateSubmittedisJsonOutputValue + session.Save(r, w) - templateParsed, errValidity := template.New("").Funcs(functions.FunctionsMap()).Parse(string(lastTemplateSubmittedValue)) + templateParsed, errValidity := template.New("").Funcs(functions.FunctionsMap()).Parse(lastTemplateSubmittedValue) if errValidity != nil { log.Println("errValidity ", errValidity) From 25b3092eabb200365ba1adba4ba3825c233e68df Mon Sep 17 00:00:00 2001 From: ugol Date: Thu, 8 Aug 2024 14:34:23 +0200 Subject: [PATCH 03/51] fixed typo --- pkg/cmd/emitterRun.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cmd/emitterRun.go b/pkg/cmd/emitterRun.go index d5e4e1c4..e5117d21 100644 --- a/pkg/cmd/emitterRun.go +++ b/pkg/cmd/emitterRun.go @@ -46,5 +46,5 @@ func RunEmitters(emitterNames []string, ems map[string][]emitter.Emitter, dryrun func init() { emitterCmd.AddCommand(emitterRunCmd) - emitterRunCmd.Flags().BoolP("dryrun", "d", false, "dryrun: output of the emitters overrode to stdout") + emitterRunCmd.Flags().BoolP("dryrun", "d", false, "dryrun: output of the emitters to stdout") } From 7eea2610678cfbaa639162e82673e7645f062487 Mon Sep 17 00:00:00 2001 From: ugol Date: Thu, 8 Aug 2024 14:46:05 +0200 Subject: [PATCH 04/51] minor refactorings --- pkg/producers/kafka/kafkaProducer.go | 158 ++++++++++++++------------- 1 file changed, 82 insertions(+), 76 deletions(-) diff --git a/pkg/producers/kafka/kafkaProducer.go b/pkg/producers/kafka/kafkaProducer.go index 5bccd242..49959e24 100644 --- a/pkg/producers/kafka/kafkaProducer.go +++ b/pkg/producers/kafka/kafkaProducer.go @@ -86,89 +86,95 @@ func (k *KafkaManager) InitializeSchemaRegistry(configFile string) { } if k.Serializer == "avro" || k.Serializer == "avro-generic" { - // verify if CSFLE must be enabled - if conf["kekName"] != "" && conf["kmsType"] != "" && conf["kmsKeyID"] != "" { - // register kms providers - awskms.Register() - azurekms.Register() - gcpkms.Register() - hcvault.Register() - encryption.Register() - - // load avro schema file: CSFLE requires schema registration - _, currentFilePath, _, _ := runtime.Caller(0) - currentDir := filepath.Dir(currentFilePath) - filePath := filepath.Join(currentDir, "../../types/"+k.TemplateType+".avsc") - file, err := os.Open(filePath) - if err != nil { - log.Fatalf("Failed to open file: %s", err) - } - defer file.Close() + verifyCSFLE(conf, k) + } - content, err := ioutil.ReadAll(file) - if err != nil { - log.Fatalf("Failed to read file: %s", err) - } - contentString := string(content) - - // check presence of PII in schema - substring := `"confluent:tags": [ "PII" ]` - normalizedJSON := normalizeWhitespace(contentString) - normalizedSubstring := normalizeWhitespace(substring) - - if strings.Contains(normalizedJSON, normalizedSubstring) { - // upper-casing the first letter of the fields --> name - required by https://pkg.go.dev/github.com/actgardner/gogen-avro#readme-naming - re := regexp.MustCompile(`"name"\s*:\s*"([^"]+)"`) - result := re.ReplaceAllStringFunc(contentString, func(match string) string { - // extract the name part after "name: " - parts := re.FindStringSubmatch(match) - fmt.Print(len(parts)) - if len(parts) > 1 { - name := parts[1] - // capitalize the first letter of the name - capitalized := capitalizeFirstLetter(name) - // replace the original match with the new capitalized version - return "\"name\":" + "\"" + capitalized + "\"" - } - return match - }) - - // register the avro schema adding rule set: PII - schema := schemaregistry.SchemaInfo{ - Schema: result, - SchemaType: "AVRO", - RuleSet: &schemaregistry.RuleSet{ - DomainRules: []schemaregistry.Rule{ - { - Name: "encryptPII", - Kind: "TRANSFORM", - Mode: "WRITEREAD", - Type: "ENCRYPT", - Tags: []string{"PII"}, - Params: map[string]string{ - "encrypt.kek.name": conf["kekName"], - "encrypt.kms.type": conf["kmsType"], - "encrypt.kms.key.id": conf["kmsKeyID"], - }, - OnFailure: "ERROR,NONE", + k.schemaRegistry = true +} + +func verifyCSFLE(conf map[string]string, k *KafkaManager) { + if conf["kekName"] != "" && conf["kmsType"] != "" && conf["kmsKeyID"] != "" { + registerProviders() + + // load avro schema file: CSFLE requires schema registration + _, currentFilePath, _, _ := runtime.Caller(0) + currentDir := filepath.Dir(currentFilePath) + filePath := filepath.Join(currentDir, "../../types/"+k.TemplateType+".avsc") + file, err := os.Open(filePath) + if err != nil { + log.Fatalf("Failed to open file: %s", err) + } + defer file.Close() + + content, err := ioutil.ReadAll(file) + if err != nil { + log.Fatalf("Failed to read file: %s", err) + } + contentString := string(content) + + // check presence of PII in schema + substring := `"confluent:tags": [ "PII" ]` + normalizedJSON := normalizeWhitespace(contentString) + normalizedSubstring := normalizeWhitespace(substring) + + if strings.Contains(normalizedJSON, normalizedSubstring) { + // upper-casing the first letter of the fields --> name - required by https://pkg.go.dev/github.com/actgardner/gogen-avro#readme-naming + re := regexp.MustCompile(`"name"\s*:\s*"([^"]+)"`) + result := re.ReplaceAllStringFunc(contentString, func(match string) string { + // extract the name part after "name: " + parts := re.FindStringSubmatch(match) + fmt.Print(len(parts)) + if len(parts) > 1 { + name := parts[1] + // capitalize the first letter of the name + capitalized := capitalizeFirstLetter(name) + // replace the original match with the new capitalized version + return "\"name\":" + "\"" + capitalized + "\"" + } + return match + }) + + // register the avro schema adding rule set: PII + schema := schemaregistry.SchemaInfo{ + Schema: result, + SchemaType: "AVRO", + RuleSet: &schemaregistry.RuleSet{ + DomainRules: []schemaregistry.Rule{ + { + Name: "encryptPII", + Kind: "TRANSFORM", + Mode: "WRITEREAD", + Type: "ENCRYPT", + Tags: []string{"PII"}, + Params: map[string]string{ + "encrypt.kek.name": conf["kekName"], + "encrypt.kms.type": conf["kmsType"], + "encrypt.kms.key.id": conf["kmsKeyID"], }, + OnFailure: "ERROR,NONE", }, }, - } - _, err = k.schema.Register(k.Topic+"-value", schema, true) - if err != nil { - fmt.Printf("Failed to register schema: %s\n", err) - os.Exit(1) - } - - k.fleEnabled = true - + }, } + _, err = k.schema.Register(k.Topic+"-value", schema, true) + if err != nil { + fmt.Printf("Failed to register schema: %s\n", err) + os.Exit(1) + } + + k.fleEnabled = true } + } +} - k.schemaRegistry = true +func registerProviders() { + awskms.Register() + azurekms.Register() + gcpkms.Register() + hcvault.Register() + encryption.Register() } func (k *KafkaManager) Close() error { @@ -367,6 +373,6 @@ func capitalizeFirstLetter(s string) string { } func normalizeWhitespace(s string) string { - re := regexp.MustCompile(`\s+`) - return re.ReplaceAllString(s, " ") + re := regexp.MustCompile(`\s+`) + return re.ReplaceAllString(s, " ") } From 0cda5e4b9e5762d75e2dda47db1e15b7e4368848 Mon Sep 17 00:00:00 2001 From: ugol Date: Thu, 8 Aug 2024 14:55:38 +0200 Subject: [PATCH 05/51] Added experimental KEY in Value --- pkg/emitter/emitter.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/emitter/emitter.go b/pkg/emitter/emitter.go index 3332525e..44985d38 100644 --- a/pkg/emitter/emitter.go +++ b/pkg/emitter/emitter.go @@ -140,7 +140,12 @@ func (e *Emitter) Run(num int, o any) { for i := 0; i < num; i++ { k := e.KTpl.Execute() v := e.VTpl.Execute() - e.Producer.Produce([]byte(k), []byte(v), o) + kInValue := functions.GetV("KEY") + if kInValue == "" { + e.Producer.Produce([]byte(kInValue), []byte(v), o) + } else { + e.Producer.Produce([]byte(k), []byte(v), o) + } ctx.JrContext.GeneratedObjects++ ctx.JrContext.GeneratedBytes += int64(len(v)) } From edfd8b674c67b2d152a1162f2e3e7a0569bbba61 Mon Sep 17 00:00:00 2001 From: ugol Date: Thu, 8 Aug 2024 15:48:36 +0200 Subject: [PATCH 06/51] added key in value example --- pkg/emitter/emitter.go | 5 ++++- pkg/emitter/loop.go | 8 +++++++- templates/user_with_key.tpl | 7 +++++++ 3 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 templates/user_with_key.tpl diff --git a/pkg/emitter/emitter.go b/pkg/emitter/emitter.go index 44985d38..5051163d 100644 --- a/pkg/emitter/emitter.go +++ b/pkg/emitter/emitter.go @@ -138,16 +138,19 @@ func (e *Emitter) Initialize(conf configuration.GlobalConfiguration) { func (e *Emitter) Run(num int, o any) { for i := 0; i < num; i++ { + k := e.KTpl.Execute() v := e.VTpl.Execute() kInValue := functions.GetV("KEY") - if kInValue == "" { + + if kInValue != "" { e.Producer.Produce([]byte(kInValue), []byte(v), o) } else { e.Producer.Produce([]byte(k), []byte(v), o) } ctx.JrContext.GeneratedObjects++ ctx.JrContext.GeneratedBytes += int64(len(v)) + } } diff --git a/pkg/emitter/loop.go b/pkg/emitter/loop.go index d5416e4d..ca1cb988 100644 --- a/pkg/emitter/loop.go +++ b/pkg/emitter/loop.go @@ -137,7 +137,13 @@ func doTemplate(emitter Emitter) { if emitter.Oneline { v = strings.ReplaceAll(v, "\n", "") } - emitter.Producer.Produce([]byte(k), []byte(v), nil) + kInValue := functions.GetV("KEY") + + if (kInValue) != "" { + emitter.Producer.Produce([]byte(kInValue), []byte(v), nil) + } else { + emitter.Producer.Produce([]byte(k), []byte(v), nil) + } ctx.JrContext.GeneratedObjects++ ctx.JrContext.GeneratedBytes += int64(len(v)) diff --git a/templates/user_with_key.tpl b/templates/user_with_key.tpl new file mode 100644 index 00000000..29d947e4 --- /dev/null +++ b/templates/user_with_key.tpl @@ -0,0 +1,7 @@ +{{$userid := (print "user_" (counter "user_id" 1 1 ) )}}{{set_v "KEY" $userid }} +{ + "registertime": {{integer64 1487715775521 1519273364600}}, + "userid": "{{$userid}}", + "regionid": "Region_{{integer 1 9}}", + "gender": "{{randoms "MALE|FEMALE|OTHER"}}" +} \ No newline at end of file From ceee8b94ddf720081b0c7ccb6fde328886f3a054 Mon Sep 17 00:00:00 2001 From: ugo landini Date: Sat, 10 Aug 2024 11:24:35 +0200 Subject: [PATCH 07/51] Update Readme.md with latest docker link --- Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Readme.md b/Readme.md index c9a794a5..fe34344c 100644 --- a/Readme.md +++ b/Readme.md @@ -10,7 +10,7 @@ JR is a CLI program that helps you to stream quality random data for your applic ![Build](https://github.com/ugol/jr/actions/workflows/go-mac.yml/badge.svg) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![Go Reference](https://pkg.go.dev/badge/github.com/ugol/jr.svg)](https://pkg.go.dev/github.com/ugol/jr) -[![Docker](https://img.shields.io/badge/docker-latest-blue.svg)](https://hub.docker.com/r/ugol/jr) +[![Docker](https://img.shields.io/badge/docker-latest-blue.svg)](https://hub.docker.com/r/jrndio/jr) ![JR-simple](https://user-images.githubusercontent.com/89472/229626362-70ddc95d-1090-4746-a20a-fbffba4193cd.gif) From 02fcc58b07d58e04ce98f313955b78fe640ec409 Mon Sep 17 00:00:00 2001 From: Vincenzo Date: Sun, 11 Aug 2024 18:20:23 +0200 Subject: [PATCH 08/51] feat: add xk6-exec support --- Dockerfile.scratch | 39 ++++++++++++++++++ k6/exec/Dockerfile | 24 +++++++++++ k6/exec/README.md | 93 +++++++++++++++++++++++++++++++++++++++++++ k6/exec/k8s-test.yaml | 32 +++++++++++++++ 4 files changed, 188 insertions(+) create mode 100644 Dockerfile.scratch create mode 100644 k6/exec/Dockerfile create mode 100644 k6/exec/README.md create mode 100644 k6/exec/k8s-test.yaml diff --git a/Dockerfile.scratch b/Dockerfile.scratch new file mode 100644 index 00000000..921db8fa --- /dev/null +++ b/Dockerfile.scratch @@ -0,0 +1,39 @@ +FROM golang:1.22-alpine AS builder +MAINTAINER Ugo Landini + +ARG VERSION=0.3.9 +ARG USER=$(id -u -n) +ARG TIME=$(date) + +RUN apk update \ + && apk add --no-cache git ca-certificates \ + && apk add --update gcc musl-dev libssl3 librdkafka-dev pkgconf \ + && update-ca-certificates + +RUN adduser \ + --disabled-password \ + --gecos "" \ + --home "/home/jr" \ + --shell "/bin/sh" \ + --uid "100001" \ + "jr-user" + +WORKDIR /go/src/github.com/ugol/jr +COPY . . + +RUN go get -u -d -v +RUN CGO_ENABLED=1 GOOS=linux go build \ + -tags musl -v \ + -ldflags="-X 'github.com/ugol/jr/cmd.Version=${VERSION}' -X 'github.com/ugol/jr/cmd.BuildUser=${USER}' -X 'github.com/ugol/jr/cmd.BuildTime=${TIME}' -linkmode external -w -s -extldflags '-static'" \ + -a -o build/jr jr.go + +FROM scratch +COPY --from=builder /etc/passwd /etc/passwd +COPY --from=builder /etc/group /etc/group +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=builder /go/src/github.com/ugol/jr/templates/ /home/jr/.jr/templates/ +COPY --from=builder /go/src/github.com/ugol/jr/config/ /home/jr/.jr/ +COPY --from=builder /go/src/github.com/ugol/jr/pkg/producers/kafka/*.examples /home/jr/.jr/kafka/ +COPY --from=builder /go/src/github.com/ugol/jr/build/jr /bin/jr + +USER jr-user:jr-user diff --git a/k6/exec/Dockerfile b/k6/exec/Dockerfile new file mode 100644 index 00000000..32e9cc6b --- /dev/null +++ b/k6/exec/Dockerfile @@ -0,0 +1,24 @@ +FROM jr-alpine as jr +FROM golang:1.20 as builder + +# build with exec extension +RUN go install go.k6.io/xk6/cmd/xk6@latest && \ + xk6 build \ + --with github.com/grafana/xk6-exec@latest \ + --output /k6 + + +# build jr + + +# build from grafana +FROM grafana/k6:latest + +USER jr-user +WORKDIR /home/jr + +COPY --from=builder /k6 /usr/bin/k6 +COPY --from=jr /etc/passwd /etc/passwd +COPY --from=jr /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=jr /home/jr/.jr .jr +COPY --from=jr /bin/jr /usr/bin/jr diff --git a/k6/exec/README.md b/k6/exec/README.md new file mode 100644 index 00000000..3572980b --- /dev/null +++ b/k6/exec/README.md @@ -0,0 +1,93 @@ +## JR and K6 distributed in Kubernetes + + +JR can run with [xk6-exec](https://github.com/grafana/xk6-exec) with the [distributed K6](https://grafana.com/docs/k6/latest/set-up/set-up-distributed-k6/) in Kubernetes. + +### Prerequisites + +- a Kubernetes cluster +- the [K6 operator](https://grafana.com/docs/k6/latest/set-up/set-up-distributed-k6/) installed +- the `jr` image accessible either as built locally or from a registry + + +### Build the image + +From the `/k6/exec` folder launch: + +```bash +docker build -t k6-jr . +``` + +The `k6-jr` image can launch a script that executes `jr` in the runner. +e.g.: + +```javascript +import exec from 'k6/x/exec'; + +export default ()=> { + console.log(exec.command("/usr/bin/jr", ["run", + "net_device", + "-n", "2", + "-f", "100ms", + "-d", "1m" + ])) + +} +``` + +### Create the CR + +You can now create the CR from the k6 CRD to run the test. +The following example runs `jr` with a parallelism of `4` and `200` virtual users (e.g. 50 VUs each pod) + + +```yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: jr-test +data: + jr.js: | + import exec from 'k6/x/exec'; + + export const options = { + vus: 20, + duration: '1m', + }; + export default ()=> { + console.log(exec.command("/usr/bin/jr", ["run", + "net_device" + ])) + + } +--- +apiVersion: k6.io/v1alpha1 +kind: TestRun +metadata: + name: k6-jr-example +spec: + parallelism: 4 + script: + configMap: + name: jr-test + file: jr.js + runner: + image: registry.localhost:5000/k6-jr:latest +``` + + +> Note: `k6` is responsible for running `jr` with the proper number pf virtual users, duration and parallelism so `jr` in the script should be run emitting just one sample + +To run the example: + +```bash +kubectl apply -f test-jr-run.yaml +``` + +The k6 operator should launch 4 pods with `jr` runnning for the amount of time declared in the `options` variable of the `javascript` test script. + + + + + diff --git a/k6/exec/k8s-test.yaml b/k6/exec/k8s-test.yaml new file mode 100644 index 00000000..7ff4385d --- /dev/null +++ b/k6/exec/k8s-test.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: jr-test +data: + jr.js: | + import exec from 'k6/x/exec'; + + export const options = { + vus: 20, + duration: '1m', + }; + export default ()=> { + console.log(exec.command("/usr/bin/jr", ["run", + "net_device" + ])) + + } +--- +apiVersion: k6.io/v1alpha1 +kind: TestRun +metadata: + name: k6-jr-example +spec: + parallelism: 4 + script: + configMap: + name: jr-test + file: jr.js + runner: + image: registry.localhost:5000/k6-jr:latest \ No newline at end of file From 095216965bf830bf9510fb3877fa295eef518c31 Mon Sep 17 00:00:00 2001 From: eljeko Date: Mon, 12 Aug 2024 11:40:20 +0200 Subject: [PATCH 09/51] UI cleanup --- pkg/cmd/html/emitters.html | 217 ------------------------------------- pkg/cmd/html/index.html | 11 -- 2 files changed, 228 deletions(-) delete mode 100644 pkg/cmd/html/emitters.html diff --git a/pkg/cmd/html/emitters.html b/pkg/cmd/html/emitters.html deleted file mode 100644 index 9a9926a1..00000000 --- a/pkg/cmd/html/emitters.html +++ /dev/null @@ -1,217 +0,0 @@ - - - - - - - JR Emitters - - - - - - - - - -
- JR

Emitters

-
-
- - -
- -
-
-
- Home -
-
- - - - - - - \ No newline at end of file diff --git a/pkg/cmd/html/index.html b/pkg/cmd/html/index.html index ebe88c03..a4d04c44 100644 --- a/pkg/cmd/html/index.html +++ b/pkg/cmd/html/index.html @@ -24,17 +24,6 @@

Web UI

- -
-
-
Emitters web UI
-

First (alpha) version to manage the configured JR Emitters

-
-
- Go to Emitters -
-
-
From 64fbf0e94154c215ff2dc38ecbc2895d545b667f Mon Sep 17 00:00:00 2001 From: eljeko Date: Mon, 12 Aug 2024 11:45:27 +0200 Subject: [PATCH 10/51] Fix after Emitters UI removed --- pkg/cmd/server.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pkg/cmd/server.go b/pkg/cmd/server.go index 3613aabf..62a6f320 100644 --- a/pkg/cmd/server.go +++ b/pkg/cmd/server.go @@ -27,9 +27,6 @@ import ( //go:embed html/index.html var index_html string -//go:embed html/emitters.html -var emitters_html string - //go:embed html/templatedev.html var templatedev_html string @@ -156,10 +153,6 @@ func embeddedFileRoutes(router chi.Router) { w.Write([]byte(index_html)) }) - router.Get("/emitters.html", func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(emitters_html)) - }) - router.Get("/templatedev.html", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(templatedev_html)) }) From 03da6bfcb930ed27c1af159fe9b9ef5c528c579d Mon Sep 17 00:00:00 2001 From: ugol Date: Mon, 12 Aug 2024 14:37:36 +0200 Subject: [PATCH 11/51] removed outdated readme-use-emitter --- README-use-emitter-templates.md | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 README-use-emitter-templates.md diff --git a/README-use-emitter-templates.md b/README-use-emitter-templates.md deleted file mode 100644 index 3eceab53..00000000 --- a/README-use-emitter-templates.md +++ /dev/null @@ -1,14 +0,0 @@ -# Emitter usage guide - -This is a list of `jr` commands to run the templates that are linked together using emitters. - -The preconfigured emitters examples have Kafka as output. - -# Global templates - -These templates: - -* util_userid.tpl -* util_ip.tpl - -Are used to preload lists of values used in the other templates in the emitter configuration. \ No newline at end of file From ac75b359c585e06ab066af0f8f202d4a639490d8 Mon Sep 17 00:00:00 2001 From: Vincenzo Date: Tue, 13 Aug 2024 10:07:17 +0200 Subject: [PATCH 12/51] feat: use k6-exec specific tag, improve js --- k6/exec/Dockerfile | 2 +- k6/exec/k8s-test.yaml | 47 ++++++++++++++++++++++++++++++++++++------- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/k6/exec/Dockerfile b/k6/exec/Dockerfile index 32e9cc6b..1e8fef91 100644 --- a/k6/exec/Dockerfile +++ b/k6/exec/Dockerfile @@ -4,7 +4,7 @@ FROM golang:1.20 as builder # build with exec extension RUN go install go.k6.io/xk6/cmd/xk6@latest && \ xk6 build \ - --with github.com/grafana/xk6-exec@latest \ + --with github.com/grafana/xk6-exec@d9fdfce4ac85e6f0c42100794b0e51047f5dd0f6 \ --output /k6 diff --git a/k6/exec/k8s-test.yaml b/k6/exec/k8s-test.yaml index 7ff4385d..987384e0 100644 --- a/k6/exec/k8s-test.yaml +++ b/k6/exec/k8s-test.yaml @@ -2,19 +2,39 @@ apiVersion: v1 kind: ConfigMap metadata: - name: jr-test + name: jr-configmap data: + mongoconfig.json: | + { + "mongo_uri": "mongodb+srv://jr:jrpassword@example-mongodb-svc.jr-test.svc.cluster.local/admin?replicaSet=example-mongodb&ssl=false", + "database": "admin", + "collection": "jr" + } jr.js: | import exec from 'k6/x/exec'; export const options = { vus: 20, - duration: '1m', + duration: '5m', + cloud: { + projectID: 3708988, + name: "jr-test-01" + } }; export default ()=> { - console.log(exec.command("/usr/bin/jr", ["run", - "net_device" - ])) + try{ + var output = exec.command("/usr/bin/jr", ["run", + "net_device", + "--output","mongo", + "--mongoConfig", "/jrconfig/mongoconfig.json" + ],{"continue_on_error": true}); + }catch (e) { + console.log("ERROR: " + e); + if (e.value && e.value.stderr) { + console.log("STDERR: " + String.fromCharCode.apply(null, e.value.stderr)) + } + } + } --- @@ -26,7 +46,20 @@ spec: parallelism: 4 script: configMap: - name: jr-test + name: jr-configmap file: jr.js + arguments: -o experimental-prometheus-rw runner: - image: registry.localhost:5000/k6-jr:latest \ No newline at end of file + image: registry.localhost:5000/k6-jr:latest + env: + - name: K6_PROMETHEUS_RW_SERVER_URL + value: http://prometheus-server.observability.svc.cluster.local:80/api/v1/write + - name: K6_PROMETHEUS_RW_TREND_STATS + value: "p(95),p(99),min,max" + volumeMounts: + - name: config-volume + mountPath: /jrconfig + volumes: + - name: config-volume + configMap: + name: jr-configmap \ No newline at end of file From 27b5ea15254f11c7ca80f4e358b88b88da40e631 Mon Sep 17 00:00:00 2001 From: Vincenzo Date: Tue, 13 Aug 2024 10:27:09 +0200 Subject: [PATCH 13/51] feat: add k6 exec distributed testing --- k6/exec/Dockerfile | 12 +++--- k6/exec/README.md | 97 ++++++++++++++++++++++++++++++++++++++++--- k6/exec/k8s-test.yaml | 27 +----------- 3 files changed, 99 insertions(+), 37 deletions(-) diff --git a/k6/exec/Dockerfile b/k6/exec/Dockerfile index 1e8fef91..c3637b83 100644 --- a/k6/exec/Dockerfile +++ b/k6/exec/Dockerfile @@ -1,23 +1,25 @@ FROM jr-alpine as jr FROM golang:1.20 as builder -# build with exec extension +# build with exec extension, using specific xk6-exec tag version to overcome the issue +# related to the missing output in case of error. +# (https://github.com/grafana/xk6-exec/issues/25#issuecomment-2259942633) RUN go install go.k6.io/xk6/cmd/xk6@latest && \ xk6 build \ --with github.com/grafana/xk6-exec@d9fdfce4ac85e6f0c42100794b0e51047f5dd0f6 \ --output /k6 -# build jr - - -# build from grafana +# build from grafana image FROM grafana/k6:latest USER jr-user WORKDIR /home/jr +# overwrite k6 with the one built in the previous stage COPY --from=builder /k6 /usr/bin/k6 + +# add jr COPY --from=jr /etc/passwd /etc/passwd COPY --from=jr /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ COPY --from=jr /home/jr/.jr .jr diff --git a/k6/exec/README.md b/k6/exec/README.md index 3572980b..b71cfd3c 100644 --- a/k6/exec/README.md +++ b/k6/exec/README.md @@ -53,14 +53,22 @@ data: export const options = { vus: 20, - duration: '1m', + duration: '5m', }; export default ()=> { - console.log(exec.command("/usr/bin/jr", ["run", - "net_device" - ])) - - } + try{ + var output = exec.command("/usr/bin/jr", ["run", + "net_device", + ],{"continue_on_error": true}); + }catch (e) { + console.log("ERROR: " + e); + if (e.value && e.value.stderr) { + console.log("STDERR: " + String.fromCharCode.apply(null, e.value.stderr)) + } + } + + + } --- apiVersion: k6.io/v1alpha1 kind: TestRun @@ -88,6 +96,83 @@ kubectl apply -f test-jr-run.yaml The k6 operator should launch 4 pods with `jr` runnning for the amount of time declared in the `options` variable of the `javascript` test script. +### A more complex example with MongoDB producer and Prometheus remote RW + +In this example we will run a distributed test with `jr` using as producer `MongoDB` and writing the `k6` metrics to `Prometheus`. + +The prerequisites are: + +- a reachable `MongoDB` cluster instance +- a reachable `Prometheus` instance with the remote write receiver enabled + +The `yaml` (self explanatory) of the TestRun CR is the following: + + +```yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: jr-configmap +data: + mongoconfig.json: | + { + "mongo_uri": "mongodb+srv://:@...", + "database": "", + "collection": "" + } + jr.js: | + import exec from 'k6/x/exec'; + + export const options = { + vus: 20, + duration: '5m', + }; + export default ()=> { + try{ + var output = exec.command("/usr/bin/jr", ["run", + "net_device", + "--output","mongo", + "--mongoConfig", "/jrconfig/mongoconfig.json" + ],{"continue_on_error": true}); + }catch (e) { + console.log("ERROR: " + e); + if (e.value && e.value.stderr) { + console.log("STDERR: " + String.fromCharCode.apply(null, e.value.stderr)) + } + } + + + } +--- +apiVersion: k6.io/v1alpha1 +kind: TestRun +metadata: + name: k6-jr-example +spec: + parallelism: 4 + script: + configMap: + name: jr-configmap + file: jr.js + arguments: -o experimental-prometheus-rw + runner: + image: registry.localhost:5000/k6-jr:latest + env: + - name: K6_PROMETHEUS_RW_SERVER_URL + value: http:///api/v1/write + - name: K6_PROMETHEUS_RW_TREND_STATS + value: "p(95),p(99),min,max" + volumeMounts: + - name: config-volume + mountPath: /jrconfig + volumes: + - name: config-volume + configMap: + name: jr-configmap +``` + + diff --git a/k6/exec/k8s-test.yaml b/k6/exec/k8s-test.yaml index 987384e0..a913ef3f 100644 --- a/k6/exec/k8s-test.yaml +++ b/k6/exec/k8s-test.yaml @@ -4,29 +4,17 @@ kind: ConfigMap metadata: name: jr-configmap data: - mongoconfig.json: | - { - "mongo_uri": "mongodb+srv://jr:jrpassword@example-mongodb-svc.jr-test.svc.cluster.local/admin?replicaSet=example-mongodb&ssl=false", - "database": "admin", - "collection": "jr" - } jr.js: | import exec from 'k6/x/exec'; export const options = { vus: 20, duration: '5m', - cloud: { - projectID: 3708988, - name: "jr-test-01" - } }; export default ()=> { try{ var output = exec.command("/usr/bin/jr", ["run", "net_device", - "--output","mongo", - "--mongoConfig", "/jrconfig/mongoconfig.json" ],{"continue_on_error": true}); }catch (e) { console.log("ERROR: " + e); @@ -48,18 +36,5 @@ spec: configMap: name: jr-configmap file: jr.js - arguments: -o experimental-prometheus-rw runner: - image: registry.localhost:5000/k6-jr:latest - env: - - name: K6_PROMETHEUS_RW_SERVER_URL - value: http://prometheus-server.observability.svc.cluster.local:80/api/v1/write - - name: K6_PROMETHEUS_RW_TREND_STATS - value: "p(95),p(99),min,max" - volumeMounts: - - name: config-volume - mountPath: /jrconfig - volumes: - - name: config-volume - configMap: - name: jr-configmap \ No newline at end of file + image: registry.localhost:5000/k6-jr:latest \ No newline at end of file From 4e2ea1e215b08809da8731482aac3338652d7592 Mon Sep 17 00:00:00 2001 From: Vincenzo Date: Tue, 13 Aug 2024 16:35:09 +0200 Subject: [PATCH 14/51] feat: add locust folder --- locust/Dockerfile | 12 +++++++ locust/README.md | 87 +++++++++++++++++++++++++++++++++++++++++++++++ locust/jr.py | 41 ++++++++++++++++++++++ 3 files changed, 140 insertions(+) create mode 100644 locust/Dockerfile create mode 100644 locust/README.md create mode 100644 locust/jr.py diff --git a/locust/Dockerfile b/locust/Dockerfile new file mode 100644 index 00000000..2b01eadd --- /dev/null +++ b/locust/Dockerfile @@ -0,0 +1,12 @@ +FROM jr-alpine as jr +FROM locustio/locust as locust + +USER jr-user +WORKDIR /home/jr + + +# add jr +COPY --from=jr /etc/passwd /etc/passwd +COPY --from=jr /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=jr /home/jr/.jr .jr +COPY --from=jr /bin/jr /usr/bin/jr diff --git a/locust/README.md b/locust/README.md new file mode 100644 index 00000000..6002819e --- /dev/null +++ b/locust/README.md @@ -0,0 +1,87 @@ +## JR and Locust distributed in Kubernetes + +JR can run with [Locust K6](https://locust.io) + +### Prerequisites + +- a Kubernetes cluster +- the `jr` image accessible either as built locally or from a registry +- [`helm`](https://helm.sh) client + + +### Build the image + +Build the customised `jr` and `locust` image with: + +``` +docker build -t locust-jr . +``` + +### Create the configmap for the jr locust library + +Create a configmap with: + +``` +kubectl create configmap locust-jr-lib --from-file jr.py +``` + + +### Example with jr and the MongoDB Producer + +Create a configmap with the MongoDB configuration and the locust python file: + + +```yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mongodb-jr +data: + mongoconfig.json: | + { + "mongo_uri": "mongodb+srv://..."," + "database": "[mongodb database]", + "collection": "[mongodb collection]" + } +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: locust-jr +data: + main.py: | + from locust import task + from lib.jr import JRUser + + class MyUser(JRUser): + @task + def jr_task(self): + self.run_jr(["run", + "net_device", + "--output","mongo", + "--mongoConfig", "/jrconfig/mongoconfig.json"]) +``` + +Add the `deliveryhero` repo: + +``` +https://github.com/deliveryhero/helm-charts/tree/master/stable/locust +``` + +Install the [Locust Helm chart](https://github.com/deliveryhero/helm-charts/tree/master/stable/locust) with: + +```bash +helm install locust deliveryhero/locust \ + --set loadtest.name=jr-test-01 \ + --set loadtest.locust_locustfile_configmap=locust-jr \ + --set loadtest.locust_lib_configmap=locust-jr-lib \ + --set image.pullPolicy=Always \ + --set worker.image=registry.localhost:5000/locust-jr \ + --set worker.logLevel=DEBUG \ + --set worker.replicas=4 \ + --set extraConfigMaps.mongodb-jr="/jrconfig" +``` + +Expose the port `8089` of the locust master pod via port forward, connect with the browser to http://localhost:8089 and launch the test (ignore the Host field in the form) + diff --git a/locust/jr.py b/locust/jr.py new file mode 100644 index 00000000..6d0a40a5 --- /dev/null +++ b/locust/jr.py @@ -0,0 +1,41 @@ +""" +JR Locust library +""" +import time +import subprocess +from locust import User + + +class JRUser(User): + """ + Superclass for JR users + """ + abstract = True + + def __init__(self, environment): + """ + Init User + """ + super().__init__(environment) + self.env = environment + self.jr = "/usr/bin/jr" + + def run_jr(self, args): + """ + Runs JR + """ + start_perf_counter = time.perf_counter() + exc = None + try: + subprocess.run([self.jr]+args, check=True) + except subprocess.CalledProcessError as e: + exc = e + + self.env.events.request.fire( + request_type="jr", + name="jr", + response_time=(time.perf_counter() - start_perf_counter) * 1000, + response_length=0, + response=None, + context=None, + exception=exc) From 300b3d9e23169990aec28b89ece0a142284b3af1 Mon Sep 17 00:00:00 2001 From: Vincenzo Date: Tue, 13 Aug 2024 16:53:14 +0200 Subject: [PATCH 15/51] doc: add section on distributed testing --- Readme.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Readme.md b/Readme.md index fe34344c..79687f9a 100644 --- a/Readme.md +++ b/Readme.md @@ -141,3 +141,11 @@ jr run net_device -n 2 -f 100ms -d 1m --kcat | jq parse error: Expected value before ',' at line 1, column 5 ``` + +## Distributed Testing + +JR can be run as a synthetic data generator for distributed testing. +At present the following testing tools are supported: + +- [k6](./k6/exec/) +- [locust](./locust/) \ No newline at end of file From dcb6b1db86cc2a731945ad93209bfddca373fe72 Mon Sep 17 00:00:00 2001 From: ugo landini Date: Tue, 13 Aug 2024 21:30:28 +0200 Subject: [PATCH 16/51] Update Readme.md --- Readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Readme.md b/Readme.md index 79687f9a..a9f43c89 100644 --- a/Readme.md +++ b/Readme.md @@ -74,7 +74,7 @@ jr run net_device ### Using Docker -You can also use a [![Docker](https://img.shields.io/badge/docker-latest-blue.svg)](https://hub.docker.com/r/ugol/jr) +You can also use a [![Docker](https://img.shields.io/badge/docker-latest-blue.svg)](https://hub.docker.com/r/jrndio/jr) image if you prefer. ```bash @@ -148,4 +148,4 @@ JR can be run as a synthetic data generator for distributed testing. At present the following testing tools are supported: - [k6](./k6/exec/) -- [locust](./locust/) \ No newline at end of file +- [locust](./locust/) From ddb9d0a408b220fbc415fccd7370efcdeb47e541 Mon Sep 17 00:00:00 2001 From: ugo landini Date: Tue, 13 Aug 2024 21:31:00 +0200 Subject: [PATCH 17/51] Update Readme.md --- Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Readme.md b/Readme.md index a9f43c89..18648074 100644 --- a/Readme.md +++ b/Readme.md @@ -78,7 +78,7 @@ You can also use a [![Docker](https://img.shields.io/badge/docker-latest-blue.sv image if you prefer. ```bash -docker run -it ugol/jr:latest jr run net_device +docker run -it jrndio/jr:latest jr run net_device ``` ### Other options for templates From d3a3b6a8c5782528b031d45aa7e2e9f8ff6385c2 Mon Sep 17 00:00:00 2001 From: eljeko Date: Wed, 14 Aug 2024 15:52:13 +0200 Subject: [PATCH 18/51] Searching for function in UI now is done with regex with live results displayed as you type --- pkg/cmd/html/templatedev.html | 29 +++++++++------ pkg/cmd/server.go | 67 +++++++++++++++++++++++++++++------ 2 files changed, 76 insertions(+), 20 deletions(-) diff --git a/pkg/cmd/html/templatedev.html b/pkg/cmd/html/templatedev.html index 05839c96..265e4961 100644 --- a/pkg/cmd/html/templatedev.html +++ b/pkg/cmd/html/templatedev.html @@ -208,16 +208,25 @@
Emitter info
if (data.ok) { data.json().then(function_found => { $('#functionslist').empty(); - $('#functionslist').append(`Name: ${function_found.Name}
`); - $('#functionslist').append(`Category: ${function_found.Category}
`); - $('#functionslist').append(`Description: ${function_found.Description}
`); - if (function_found.Parameters.length > 0) { - $('#functionslist').append(`Parameters: ${function_found.Parameters}
`); - } - $('#functionslist').append(`Localizable: ${function_found.Localizable}
`); - $('#functionslist').append(`Return: ${function_found.Return}
`); - $('#functionslist').append(`Example: 
${function_found.Example}
`); - $('#functionslist').append(`Output: 
${function_found.Output}

`); + function_found.functions.forEach(aFunction => { + + + + $('#functionslist').append(`Name: ${aFunction.Name}
`); + $('#functionslist').append(`Category: ${aFunction.Category}
`); + $('#functionslist').append(`Description: ${aFunction.Description}
`); + if (aFunction.Parameters.length > 0) { + $('#functionslist').append(`Parameters: ${aFunction.Parameters}
`); + } + $('#functionslist').append(`Localizable: ${aFunction.Localizable}
`); + $('#functionslist').append(`Return: ${aFunction.Return}
`); + $('#functionslist').append(`Example: 
${aFunction.Example}
`); + $('#functionslist').append(`Output: 
${aFunction.Output}

`); + + + }); + + }) } else if (!data.ok) { diff --git a/pkg/cmd/server.go b/pkg/cmd/server.go index 62a6f320..2c79055e 100644 --- a/pkg/cmd/server.go +++ b/pkg/cmd/server.go @@ -10,6 +10,8 @@ import ( "log" "net/http" "path/filepath" + "regexp" + "sort" "strings" "text/template" "time" @@ -436,25 +438,70 @@ func webFunctionList(w http.ResponseWriter, r *http.Request) { } func webPrintFunction(web_function_to_find string, w http.ResponseWriter, r *http.Request) { - f, found := functions.Description(web_function_to_find) - if found { - b, errMarshal := json.Marshal(f) - if errMarshal != nil { - fmt.Println(errMarshal) - return - } + matchingFunction := findFunctonsByRegex(web_function_to_find) + + sort.Strings(matchingFunction) + + if len(matchingFunction) > 0 { + w.Header().Set("Content-Type", "application/json") - _, err := w.Write(b) - if err != nil { - log.Println(err) + _, err := w.Write([]byte("{\"functions\":[")) + + for i, function_name := range matchingFunction { + + f, _ := functions.Description(function_name) + + b, errMarshal := json.Marshal(f) + + if errMarshal != nil { + fmt.Println(errMarshal) + return + } + + if err != nil { + log.Println(err) + } + + _, err = w.Write(b) + + if i < len(matchingFunction)-1 { + _, err = w.Write([]byte(",")) + } + + if err != nil { + log.Println(err) + } } + + _, err = w.Write([]byte("]}")) + } else { http.Error(w, "No function found", http.StatusNotFound) } } +func findFunctonsByRegex(name string) []string { + var matchedKeys []string + + // Compile the regular expression pattern + re, err := regexp.Compile(name) + if err != nil { + fmt.Println("Invalid regex pattern:", err) + return nil + } + + // Iterate over the map and match the keys against the regex pattern + for key := range functions.DescriptionMap() { + if re.MatchString(key) { + matchedKeys = append(matchedKeys, key) + } + } + + return matchedKeys +} + func init() { rootCmd.AddCommand(serverCmd) serverCmd.Flags().IntP("port", "p", constants.DEFAULT_HTTP_PORT, "Server port") From 7c1993fa47ae431ef14068d3561c640c11f72a43 Mon Sep 17 00:00:00 2001 From: Vincenzo Date: Thu, 15 Aug 2024 18:52:55 +0200 Subject: [PATCH 19/51] feat: add http producer --- go.mod | 11 +-- go.sum | 41 ++++++--- pkg/cmd/producerList.go | 2 + pkg/cmd/templateRun.go | 6 +- pkg/configuration/configuration.go | 1 + pkg/emitter/emitter.go | 22 ++++- pkg/producers/http/config.go | 70 ++++++++++++++++ pkg/producers/http/producer.go | 128 +++++++++++++++++++++++++++++ 8 files changed, 261 insertions(+), 20 deletions(-) create mode 100644 pkg/producers/http/config.go create mode 100644 pkg/producers/http/producer.go diff --git a/go.mod b/go.mod index 195f685a..9ee33c64 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,9 @@ require ( github.com/confluentinc/confluent-kafka-go/v2 v2.5.0 github.com/elastic/go-elasticsearch/v8 v8.14.0 github.com/go-chi/chi/v5 v5.1.0 + github.com/go-resty/resty/v2 v2.14.0 github.com/google/uuid v1.6.0 + github.com/gorilla/sessions v1.3.0 github.com/redis/go-redis/v9 v9.5.3 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 @@ -60,7 +62,6 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.5 // indirect github.com/gorilla/securecookie v1.1.2 // indirect - github.com/gorilla/sessions v1.3.0 // indirect github.com/hamba/avro/v2 v2.20.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect @@ -100,12 +101,12 @@ require ( go.opentelemetry.io/otel/metric v1.24.0 // indirect go.opentelemetry.io/otel/trace v1.24.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.24.0 // indirect + golang.org/x/crypto v0.25.0 // indirect golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 // indirect - golang.org/x/net v0.26.0 // indirect + golang.org/x/net v0.27.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/time v0.5.0 // indirect + golang.org/x/time v0.6.0 // indirect google.golang.org/api v0.187.0 // indirect google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect @@ -130,7 +131,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/squeeze69/codicefiscale v1.0.4 // indirect github.com/subosito/gotenv v1.6.0 // indirect - golang.org/x/sys v0.21.0 // indirect + golang.org/x/sys v0.22.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index d2679108..d0b8964a 100644 --- a/go.sum +++ b/go.sum @@ -190,6 +190,8 @@ github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2Kv github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-resty/resty/v2 v2.14.0 h1:/rhkzsAqGQkozwfKS5aFAbb6TyKd3zyFRWcdRXLPCAU= +github.com/go-resty/resty/v2 v2.14.0/go.mod h1:IW6mekUOsElt9C7oWr0XRt9BNSD6D5rr9mhk6NjmNHg= github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= @@ -571,9 +573,11 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 h1:/RIbNt/Zr7rVhIkQhooTxCxFcdWLGIKnZA4IXNFSrvo= golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= @@ -582,6 +586,9 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -593,8 +600,11 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= @@ -603,6 +613,8 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -617,27 +629,34 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -646,6 +665,8 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.187.0 h1:Mxs7VATVC2v7CY+7Xwm4ndkX71hpElcvx0D1Ji/p1eo= diff --git a/pkg/cmd/producerList.go b/pkg/cmd/producerList.go index 411f05d3..ce4e5704 100644 --- a/pkg/cmd/producerList.go +++ b/pkg/cmd/producerList.go @@ -22,6 +22,7 @@ package cmd import ( "fmt" + "github.com/spf13/cobra" ) @@ -47,6 +48,7 @@ var producerListCmd = &cobra.Command{ fmt.Printf("%sConsole *%s (--output = stdout)\n", Green, Reset) fmt.Printf("%sKafka%s (--output = kafka)\n", Green, Reset) + fmt.Printf("%sHTTP%s (--output = http)\n", Green, Reset) fmt.Printf("%sRedis%s (--output = redis)\n", Green, Reset) fmt.Printf("%sMongodb%s (--output = mongo)\n", Green, Reset) fmt.Printf("%sElastic%s (--output = elastic)\n", Green, Reset) diff --git a/pkg/cmd/templateRun.go b/pkg/cmd/templateRun.go index 2300f11d..b81ea758 100644 --- a/pkg/cmd/templateRun.go +++ b/pkg/cmd/templateRun.go @@ -21,13 +21,14 @@ package cmd import ( + "time" + "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/ugol/jr/pkg/configuration" "github.com/ugol/jr/pkg/constants" "github.com/ugol/jr/pkg/emitter" "github.com/ugol/jr/pkg/functions" - "time" ) var templateRunCmd = &cobra.Command{ @@ -109,6 +110,8 @@ jr template run --template "{{name}}" configuration.GlobalCfg.S3Config, _ = cmd.Flags().GetString(f.Name) case "gcsConfig": configuration.GlobalCfg.GCSConfig, _ = cmd.Flags().GetString(f.Name) + case "httpConfig": + configuration.GlobalCfg.HTTPConfig, _ = cmd.Flags().GetString(f.Name) } } }) @@ -166,6 +169,7 @@ func init() { templateRunCmd.Flags().BoolP("schemaRegistry", "s", false, "If you want to use Confluent Schema Registry") templateRunCmd.Flags().String("serializer", "", "Type of serializer: json-schema, avro-generic, avro, protobuf") templateRunCmd.Flags().Duration("redis.ttl", -1, "If output is redis, ttl of the object") + templateRunCmd.Flags().String("httpConfig", "", "HTTP configuration") templateRunCmd.Flags().String("redisConfig", "", "Redis configuration") templateRunCmd.Flags().String("mongoConfig", "", "MongoDB configuration") templateRunCmd.Flags().String("elasticConfig", "", "Elastic Search configuration") diff --git a/pkg/configuration/configuration.go b/pkg/configuration/configuration.go index f436b060..b256c622 100644 --- a/pkg/configuration/configuration.go +++ b/pkg/configuration/configuration.go @@ -37,6 +37,7 @@ type GlobalConfiguration struct { ElasticConfig string S3Config string GCSConfig string + HTTPConfig string Url string EmbeddedTemplate bool FileNameTemplate bool diff --git a/pkg/emitter/emitter.go b/pkg/emitter/emitter.go index 5051163d..c6cddfd8 100644 --- a/pkg/emitter/emitter.go +++ b/pkg/emitter/emitter.go @@ -22,6 +22,10 @@ package emitter import ( "fmt" + "log" + "os" + "time" + "github.com/ugol/jr/pkg/configuration" "github.com/ugol/jr/pkg/constants" "github.com/ugol/jr/pkg/ctx" @@ -29,15 +33,13 @@ import ( "github.com/ugol/jr/pkg/producers/console" "github.com/ugol/jr/pkg/producers/elastic" "github.com/ugol/jr/pkg/producers/gcs" + "github.com/ugol/jr/pkg/producers/http" "github.com/ugol/jr/pkg/producers/kafka" "github.com/ugol/jr/pkg/producers/mongoDB" "github.com/ugol/jr/pkg/producers/redis" "github.com/ugol/jr/pkg/producers/s3" "github.com/ugol/jr/pkg/producers/server" "github.com/ugol/jr/pkg/tpl" - "log" - "os" - "time" ) type Emitter struct { @@ -128,11 +130,16 @@ func (e *Emitter) Initialize(conf configuration.GlobalConfiguration) { return } - if e.Output == "http" { + if e.Output == "json" { e.Producer = &server.JsonProducer{OutputTpl: &o} return } + if e.Output == "http" { + e.Producer = createHTTPProducer(conf.HTTPConfig) + return + } + } func (e *Emitter) Run(num int, o any) { @@ -191,6 +198,13 @@ func createGCSProducer(gcsConfig string) Producer { return gProducer } +func createHTTPProducer(httpConfig string) Producer { + httpProducer := &http.Producer{} + httpProducer.Initialize(httpConfig) + + return httpProducer +} + func createKafkaProducer(conf configuration.GlobalConfiguration, topic string, templateType string) *kafka.KafkaManager { kManager := &kafka.KafkaManager{ diff --git a/pkg/producers/http/config.go b/pkg/producers/http/config.go new file mode 100644 index 00000000..d06368c6 --- /dev/null +++ b/pkg/producers/http/config.go @@ -0,0 +1,70 @@ +package http + +import "time" + +type AuthType string +type Method string + +const ( + BasicAuth AuthType = "basic" + BearerAuth AuthType = "bearer" + APIKeyAuth AuthType = "api_key" + DigestAuth AuthType = "digest" + + POST Method = "POST" + PUT Method = "PUT" +) + +type Endpoint struct { + URL string `json:"url"` + Method Method `json:"method"` + Timeout time.Duration `json:"timeout"` +} + +type Session struct { + UseCookieJar bool `json:"use_cookie_jar"` +} + +type ErrorHandling struct { + ExpectStatusCode int `json:"expect_status_code"` + IgnoreStatusCode bool `json:"ignore_status_code"` + // MaxErrors int `json:"max_errors"` +} + +type Headers map[string]string +type TLS struct { + InsecureSkipVerify bool `json:"insecure_skip_verify"` + CertFile string `json:"cert_file"` + KeyFile string `json:"key_file"` + RootCAFile string `json:"root_ca_file"` +} + +type APIKey struct { + Header string `json:"header"` + Value string `json:"Value"` +} + +type Bearer struct { + Token string `json:"token"` +} + +type Basic struct { + Username string `json:"username"` + Password string `json:"password"` +} +type Authentication struct { + Type AuthType `json:"type"` + Basic Basic `json:"basic"` + Digest Basic `json:"digest"` + Bearer Bearer `json:"bearer"` + APIKey APIKey `json:"api_key"` +} + +type Config struct { + Endpoint Endpoint `json:"endpoint"` + Session Session `json:"session"` + ErrorHandling ErrorHandling `json:"error_handling"` + Headers Headers `json:"headers"` + TLS TLS `json:"tls"` + Authentication Authentication `json:"authentication"` +} diff --git a/pkg/producers/http/producer.go b/pkg/producers/http/producer.go new file mode 100644 index 00000000..b918a6ac --- /dev/null +++ b/pkg/producers/http/producer.go @@ -0,0 +1,128 @@ +package http + +import ( + "crypto/tls" + "encoding/json" + "log" + "net/http" + "net/http/cookiejar" + "os" + "time" + + "github.com/go-resty/resty/v2" +) + +type Producer struct { + configuration Config + + certificate tls.Certificate + client *resty.Client + cookiejar http.CookieJar +} + +func (p *Producer) Initialize(configFile string) { + cfgBytes, err := os.ReadFile(configFile) + if err != nil { + log.Fatalf("Failed to read config file: %v", err) + } + + config := Config{} + if err := json.Unmarshal(cfgBytes, &config); err != nil { + log.Fatalf("Failed to unmarshal config: %v", err) + } + + p.configuration = config + if p.configuration.Endpoint.Timeout == 0 { + p.configuration.Endpoint.Timeout = time.Second * 10 + } + + if p.configuration.TLS.CertFile != "" && p.configuration.TLS.KeyFile == "" { + log.Fatalf("CertFile is set but KeyFile is not") + } + if p.configuration.TLS.CertFile == "" && p.configuration.TLS.KeyFile != "" { + log.Fatalf("KeyFile is set but CertFile is not") + } + + if p.configuration.TLS.CertFile != "" { + p.certificate, err = tls.LoadX509KeyPair(p.configuration.TLS.CertFile, p.configuration.TLS.KeyFile) + if err != nil { + log.Fatalf("Failed to load certificate: %v", err) + } + } + + if p.configuration.Session.UseCookieJar { + p.cookiejar, err = cookiejar.New(nil) + if err != nil { + log.Fatalf("Failed to create cookie jar: %v", err) + } + } + + p.client = resty.New(). + SetTimeout(p.configuration.Endpoint.Timeout). + SetTLSClientConfig(&tls.Config{ + InsecureSkipVerify: p.configuration.TLS.InsecureSkipVerify, + }). + SetHeaders(p.configuration.Headers) + + if p.configuration.Session.UseCookieJar { + p.client.SetCookieJar(p.cookiejar) + } + + if p.configuration.TLS.CertFile != "" { + p.client.SetCertificates(p.certificate) + } + + if p.configuration.TLS.RootCAFile != "" { + p.client.SetRootCertificate(p.configuration.TLS.RootCAFile) + } + + switch p.configuration.Authentication.Type { + case BasicAuth: + p.client.SetBasicAuth(p.configuration.Authentication.Basic.Username, + p.configuration.Authentication.Basic.Password) + case BearerAuth: + p.client.SetAuthToken(p.configuration.Authentication.Bearer.Token) + case APIKeyAuth: + p.client.SetHeader(p.configuration.Authentication.APIKey.Header, + p.configuration.Authentication.APIKey.Value) + case DigestAuth: + p.client.SetDigestAuth(p.configuration.Authentication.Digest.Username, + p.configuration.Authentication.Digest.Password) + default: + + } + +} + +func (p *Producer) Produce(k []byte, v []byte, o any) { + + var err error + + // creating request + req := p.client.R(). + SetBody(v) + + resp := &resty.Response{} + switch p.configuration.Endpoint.Method { + case POST: + resp, err = req.Post(p.configuration.Endpoint.URL) + case PUT: + resp, err = req.Put(p.configuration.Endpoint.URL) + default: + resp, err = req.Post(p.configuration.Endpoint.URL) + } + + if err != nil { + log.Fatalf("Failed to send request: %v", err) + } + + if resp.StatusCode() != p.configuration.ErrorHandling.ExpectStatusCode && + !p.configuration.ErrorHandling.IgnoreStatusCode { + log.Fatalf("Unexpected status code: %d", resp.StatusCode()) + } + +} + +func (p *Producer) Close() error { + return nil +} From 1ca5589b64431704394ec8e6364e9d1af2cf4ca4 Mon Sep 17 00:00:00 2001 From: ugol Date: Fri, 16 Aug 2024 08:56:15 +0200 Subject: [PATCH 20/51] fixed typo --- locust/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/locust/README.md b/locust/README.md index 6002819e..bf4eb12b 100644 --- a/locust/README.md +++ b/locust/README.md @@ -1,6 +1,6 @@ ## JR and Locust distributed in Kubernetes -JR can run with [Locust K6](https://locust.io) +JR can run with [Locust](https://locust.io) ### Prerequisites From 160f7f464e9d0ca565e7bbd5f909c512096d890f Mon Sep 17 00:00:00 2001 From: ugol Date: Fri, 16 Aug 2024 10:03:40 +0200 Subject: [PATCH 21/51] added go version to binary --- Makefile | 3 ++- pkg/cmd/version.go | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index cc6bdf5b..b3819cb4 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ VERSION=0.3.9 +GOVERSION=$(shell go version) USER=$(shell id -u -n) TIME=$(shell date) @@ -14,7 +15,7 @@ generate: compile: @echo "Compiling" - go build -v -ldflags="-X 'github.com/ugol/jr/pkg/cmd.Version=$(VERSION)' -X 'github.com/ugol/jr/pkg/cmd.BuildUser=$(USER)' -X 'github.com/ugol/jr/pkg/cmd.BuildTime=$(TIME)'" -o build/jr jr.go + go build -v -ldflags="-X 'github.com/ugol/jr/pkg/cmd.Version=$(VERSION)' -X 'github.com/ugol/jr/pkg/cmd.GoVersion=$(GOVERSION)' -X 'github.com/ugol/jr/pkg/cmd.BuildUser=$(USER)' -X 'github.com/ugol/jr/pkg/cmd.BuildTime=$(TIME)'" -o build/jr jr.go run: compile ./build/jr diff --git a/pkg/cmd/version.go b/pkg/cmd/version.go index bd0f4c91..bb4c945c 100644 --- a/pkg/cmd/version.go +++ b/pkg/cmd/version.go @@ -26,6 +26,7 @@ import ( ) var Version = "DEV" +var GoVersion string var BuildTime string var BuildUser string @@ -35,8 +36,9 @@ var versionCmd = &cobra.Command{ Long: `prints JR version number`, Run: func(cmd *cobra.Command, args []string) { fmt.Printf("JR Version: %s\n", Version) - fmt.Printf("Built by %s\n", BuildUser) - fmt.Printf("Build time: %s\n", BuildTime) + fmt.Printf("Built with: %s\n", GoVersion) + fmt.Printf(" by %s\n", BuildUser) + fmt.Printf(" at: %s\n", BuildTime) }, } From 1af617607caded38d303e60629774466336d46eb Mon Sep 17 00:00:00 2001 From: ugol Date: Fri, 16 Aug 2024 10:07:25 +0200 Subject: [PATCH 22/51] added go version to binary in dockerfiles --- Dockerfile | 3 ++- Dockerfile.alpine | 3 ++- Dockerfile.scratch | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6810c73d..3c590c29 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,7 @@ FROM golang AS builder MAINTAINER Ugo Landini ARG VERSION=0.3.9 +ARG GOVERSION=$(go version) ARG USER=$(id -u -n) ARG TIME=$(date) @@ -12,7 +13,7 @@ COPY . . RUN go install github.com/actgardner/gogen-avro/v10/cmd/...@latest RUN go generate pkg/generator/generate.go RUN go get -u -d -v -RUN CGO_ENABLED=1 GOOS=linux go build -tags static_all -v -ldflags="-X 'github.com/ugol/jr/pkg/cmd.Version=${VERSION}' -X 'github.com/ugol/jr/pkg/cmd.BuildUser=${USER}' -X 'github.com/ugol/jr/pkg/cmd.BuildTime=${TIME}'" -o build/jr jr.go +RUN CGO_ENABLED=1 GOOS=linux go build -tags static_all -v -ldflags="-X 'github.com/ugol/jr/pkg/cmd.Version=${VERSION}' -X 'github.com/ugol/jr/pkg/cmd.GoVersion=${GOVERSION}' -X 'github.com/ugol/jr/pkg/cmd.BuildUser=${USER}' -X 'github.com/ugol/jr/pkg/cmd.BuildTime=${TIME}'" -o build/jr jr.go FROM registry.access.redhat.com/ubi9/ubi-micro diff --git a/Dockerfile.alpine b/Dockerfile.alpine index 266bfb98..163bcfca 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -2,6 +2,7 @@ FROM golang:1.22-alpine AS builder MAINTAINER Ugo Landini ARG VERSION=0.3.9 +ARG GOVERSION= $(go version) ARG USER=$(id -u -n) ARG TIME=$(date) @@ -23,7 +24,7 @@ WORKDIR /go/src/github.com/ugol/jr COPY . . RUN go get -u -d -v -RUN CGO_ENABLED=1 GOOS=linux go build -tags musl -v -ldflags="-X 'github.com/ugol/jr/cmd.Version=${VERSION}' -X 'github.com/ugol/jr/cmd.BuildUser=${USER}' -X 'github.com/ugol/jr/cmd.BuildTime=${TIME}'" -o build/jr jr.go +RUN CGO_ENABLED=1 GOOS=linux go build -tags musl -v -ldflags="-X 'github.com/ugol/jr/cmd.Version=${VERSION}' 'github.com/ugol/jr/pkg/cmd.GoVersion=${GOVERSION}' -X 'github.com/ugol/jr/cmd.BuildUser=${USER}' -X 'github.com/ugol/jr/cmd.BuildTime=${TIME}'" -o build/jr jr.go FROM alpine COPY --from=builder /etc/passwd /etc/passwd diff --git a/Dockerfile.scratch b/Dockerfile.scratch index 921db8fa..f104f350 100644 --- a/Dockerfile.scratch +++ b/Dockerfile.scratch @@ -2,6 +2,7 @@ FROM golang:1.22-alpine AS builder MAINTAINER Ugo Landini ARG VERSION=0.3.9 +ARG GOVERSION= $(go version) ARG USER=$(id -u -n) ARG TIME=$(date) @@ -24,7 +25,7 @@ COPY . . RUN go get -u -d -v RUN CGO_ENABLED=1 GOOS=linux go build \ -tags musl -v \ - -ldflags="-X 'github.com/ugol/jr/cmd.Version=${VERSION}' -X 'github.com/ugol/jr/cmd.BuildUser=${USER}' -X 'github.com/ugol/jr/cmd.BuildTime=${TIME}' -linkmode external -w -s -extldflags '-static'" \ + -ldflags="-X 'github.com/ugol/jr/cmd.Version=${VERSION}' 'github.com/ugol/jr/pkg/cmd.GoVersion=${GOVERSION}' -X 'github.com/ugol/jr/cmd.BuildUser=${USER}' -X 'github.com/ugol/jr/cmd.BuildTime=${TIME}' -linkmode external -w -s -extldflags '-static'" \ -a -o build/jr jr.go FROM scratch From 595a8cbf65358ac5a95c0bfd2ea84fb19895b1b2 Mon Sep 17 00:00:00 2001 From: ugol Date: Fri, 16 Aug 2024 10:08:06 +0200 Subject: [PATCH 23/51] added go version to binary in dockerfiles --- Dockerfile.alpine | 2 +- Dockerfile.scratch | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.alpine b/Dockerfile.alpine index 163bcfca..3039453c 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -2,7 +2,7 @@ FROM golang:1.22-alpine AS builder MAINTAINER Ugo Landini ARG VERSION=0.3.9 -ARG GOVERSION= $(go version) +ARG GOVERSION=$(go version) ARG USER=$(id -u -n) ARG TIME=$(date) diff --git a/Dockerfile.scratch b/Dockerfile.scratch index f104f350..3636d5e7 100644 --- a/Dockerfile.scratch +++ b/Dockerfile.scratch @@ -2,7 +2,7 @@ FROM golang:1.22-alpine AS builder MAINTAINER Ugo Landini ARG VERSION=0.3.9 -ARG GOVERSION= $(go version) +ARG GOVERSION=$(go version) ARG USER=$(id -u -n) ARG TIME=$(date) From cfc2e1c5f17ee5c3d4124d0f3fb14083c7fe9a57 Mon Sep 17 00:00:00 2001 From: ugol Date: Fri, 16 Aug 2024 10:15:37 +0200 Subject: [PATCH 24/51] fixed indentation --- pkg/cmd/version.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cmd/version.go b/pkg/cmd/version.go index bb4c945c..ebbc8d1a 100644 --- a/pkg/cmd/version.go +++ b/pkg/cmd/version.go @@ -37,8 +37,8 @@ var versionCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { fmt.Printf("JR Version: %s\n", Version) fmt.Printf("Built with: %s\n", GoVersion) - fmt.Printf(" by %s\n", BuildUser) - fmt.Printf(" at: %s\n", BuildTime) + fmt.Printf(" by: %s\n", BuildUser) + fmt.Printf(" at: %s\n", BuildTime) }, } From 704cdcb48ccf3f9b32bd3c7e59d30a3f2dce4a05 Mon Sep 17 00:00:00 2001 From: ugol Date: Fri, 16 Aug 2024 10:32:50 +0200 Subject: [PATCH 25/51] updated release notes --- ReleaseNotes.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ReleaseNotes.md b/ReleaseNotes.md index 21f6d874..95439ea9 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -1,3 +1,11 @@ +v0.3.9 +- added key calculation directly from the template value +- added distributed JR with Locust and K6 +- new Docker image from scratch +- added several GUI features +- added http producer +- + v0.3.8 - moved to librdkafka 2.5.0 - added inject function From af1c8d3d24a6aa86e5bedcd63d3ef6adeceaf0c4 Mon Sep 17 00:00:00 2001 From: ugol Date: Fri, 16 Aug 2024 10:38:52 +0200 Subject: [PATCH 26/51] re-enabled experimental throughput parsing --- pkg/cmd/templateRun.go | 21 +++++++++++---------- pkg/emitter/throughput.go | 2 +- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/pkg/cmd/templateRun.go b/pkg/cmd/templateRun.go index 2300f11d..3074a09d 100644 --- a/pkg/cmd/templateRun.go +++ b/pkg/cmd/templateRun.go @@ -27,6 +27,7 @@ import ( "github.com/ugol/jr/pkg/constants" "github.com/ugol/jr/pkg/emitter" "github.com/ugol/jr/pkg/functions" + "log" "time" ) @@ -53,7 +54,7 @@ jr template run --template "{{name}}" num, _ := cmd.Flags().GetInt("num") frequency, _ := cmd.Flags().GetDuration("frequency") duration, _ := cmd.Flags().GetDuration("duration") - //throughputString, _ := cmd.Flags().GetString("throughput") + throughputString, _ := cmd.Flags().GetString("throughput") seed, _ := cmd.Flags().GetInt64("seed") topic, _ := cmd.Flags().GetString("topic") preload, _ := cmd.Flags().GetInt("preload") @@ -74,16 +75,16 @@ jr template run --template "{{name}}" vTemplate = args[0] eTemplate = "" } - /* - throughput, err := parseThroughput(throughputString) - if err != nil { - log.Panicf("Throughput format error:%v", err) - } - if throughput > 0 { - // @TODO - } - */ + throughput, err := emitter.ParseThroughput(throughputString) + if err != nil { + log.Panicf("Throughput format error:%v", err) + } + + if throughput > 0 { + // @TODO + } + cmd.Flags().VisitAll(func(f *pflag.Flag) { if f.Changed { switch f.Name { diff --git a/pkg/emitter/throughput.go b/pkg/emitter/throughput.go index 07e863f5..47c0830a 100644 --- a/pkg/emitter/throughput.go +++ b/pkg/emitter/throughput.go @@ -33,7 +33,7 @@ const ( ) //gocyclo:ignore -func parseThroughput(input string) (Throughput, error) { +func ParseThroughput(input string) (Throughput, error) { if input == "" { return -1, nil From d01e4f04355e994a4f08351ac0d34524f239c1ce Mon Sep 17 00:00:00 2001 From: Vincenzo Date: Fri, 16 Aug 2024 11:37:32 +0200 Subject: [PATCH 27/51] feat: add testing --- go.mod | 2 + go.sum | 4 + pkg/producers/http/config.go | 8 +- pkg/producers/http/config.json.example | 33 ++++ pkg/producers/http/producer.go | 30 +++- pkg/producers/http/producer_test.go | 228 +++++++++++++++++++++++++ 6 files changed, 298 insertions(+), 7 deletions(-) create mode 100644 pkg/producers/http/config.json.example create mode 100644 pkg/producers/http/producer_test.go diff --git a/go.mod b/go.mod index 9ee33c64..64f6a49e 100644 --- a/go.mod +++ b/go.mod @@ -10,8 +10,10 @@ require ( github.com/elastic/go-elasticsearch/v8 v8.14.0 github.com/go-chi/chi/v5 v5.1.0 github.com/go-resty/resty/v2 v2.14.0 + github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 github.com/gorilla/sessions v1.3.0 + github.com/jarcoal/httpmock v1.3.1 github.com/redis/go-redis/v9 v9.5.3 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 diff --git a/go.sum b/go.sum index d0b8964a..5c1149f4 100644 --- a/go.sum +++ b/go.sum @@ -301,6 +301,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI= github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= +github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww= +github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -339,6 +341,8 @@ github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebG github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04nTH68g= +github.com/maxatome/go-testdeep v1.12.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU= diff --git a/pkg/producers/http/config.go b/pkg/producers/http/config.go index d06368c6..20198e4a 100644 --- a/pkg/producers/http/config.go +++ b/pkg/producers/http/config.go @@ -16,9 +16,10 @@ const ( ) type Endpoint struct { - URL string `json:"url"` - Method Method `json:"method"` - Timeout time.Duration `json:"timeout"` + URL string `json:"url"` + Method Method `json:"method"` + Timeout string `json:"timeout"` + timeout time.Duration } type Session struct { @@ -28,7 +29,6 @@ type Session struct { type ErrorHandling struct { ExpectStatusCode int `json:"expect_status_code"` IgnoreStatusCode bool `json:"ignore_status_code"` - // MaxErrors int `json:"max_errors"` } type Headers map[string]string diff --git a/pkg/producers/http/config.json.example b/pkg/producers/http/config.json.example new file mode 100644 index 00000000..a791df5e --- /dev/null +++ b/pkg/producers/http/config.json.example @@ -0,0 +1,33 @@ +{ + "endpoint": { + "url": "https://jr.io", + "method": "POST", + Timeout: "10s" + }, + "session":{ + "use_cookie_jar": false + }, + "error_handling":{ + expect_status_code: 200, + ignore_status_code: false, + }, + "headers":{ + "header01":"value01", + "header02":"value02", + }, + "tls":{ + "insecure_skip_verify": false, + "cert_file": "/path/to/cert_file", + "key_file": "/path/to/key_file", + "root_ca_file":/path/to/root_ca_file" + }, + "authentication":{ + "type": "basic", + "basic":{ + "username": "user", + "password": "password", + } + + } + +} \ No newline at end of file diff --git a/pkg/producers/http/producer.go b/pkg/producers/http/producer.go index b918a6ac..d2ebd1c2 100644 --- a/pkg/producers/http/producer.go +++ b/pkg/producers/http/producer.go @@ -31,9 +31,25 @@ func (p *Producer) Initialize(configFile string) { log.Fatalf("Failed to unmarshal config: %v", err) } + p.InitializeFromConfig(config) +} + +func (p *Producer) InitializeFromConfig(config Config) { + + var err error p.configuration = config - if p.configuration.Endpoint.Timeout == 0 { - p.configuration.Endpoint.Timeout = time.Second * 10 + if p.configuration.Endpoint.Timeout == "" { + p.configuration.Endpoint.timeout = time.Second * 10 + } else { + p.configuration.Endpoint.timeout, err = time.ParseDuration(p.configuration.Endpoint.Timeout) + if err != nil { + log.Fatalf("Failed to parse timeout: %v", err) + } + + } + + if p.configuration.ErrorHandling.ExpectStatusCode == 0 { + p.configuration.ErrorHandling.ExpectStatusCode = 200 } if p.configuration.TLS.CertFile != "" && p.configuration.TLS.KeyFile == "" { @@ -58,7 +74,7 @@ func (p *Producer) Initialize(configFile string) { } p.client = resty.New(). - SetTimeout(p.configuration.Endpoint.Timeout). + SetTimeout(p.configuration.Endpoint.timeout). SetTLSClientConfig(&tls.Config{ InsecureSkipVerify: p.configuration.TLS.InsecureSkipVerify, }). @@ -92,6 +108,10 @@ func (p *Producer) Initialize(configFile string) { } + if p.configuration.Endpoint.Method == "" { + p.configuration.Endpoint.Method = POST + } + } func (p *Producer) Produce(k []byte, v []byte, o any) { @@ -126,3 +146,7 @@ func (p *Producer) Produce(k []byte, v []byte, o any) { func (p *Producer) Close() error { return nil } + +func (p *Producer) GetClient() *resty.Client { + return p.client +} diff --git a/pkg/producers/http/producer_test.go b/pkg/producers/http/producer_test.go new file mode 100644 index 00000000..f26545bf --- /dev/null +++ b/pkg/producers/http/producer_test.go @@ -0,0 +1,228 @@ +package http_test + +import ( + "encoding/base64" + "fmt" + "net/http" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/jarcoal/httpmock" + phttp "github.com/ugol/jr/pkg/producers/http" +) + +type mockResponder struct { + name string + t *testing.T + expectHeaders map[string]string + status int + basic string + bearer string + apikey string +} + +func (m *mockResponder) serveHTTP(req *http.Request) (*http.Response, error) { + + if m.expectHeaders != nil { + // flatten request headers + reqHeaders := make(map[string]string) + for k, v := range req.Header { + // testing only headers starting with "Test-jr" + if strings.HasPrefix(k, "Test-Jr") { + reqHeaders[k] = v[0] + } + } + + // canonical names for headers + wantHeaders := make(map[string]string) + for k, v := range m.expectHeaders { + wantHeaders[http.CanonicalHeaderKey(k)] = v + } + if diff := cmp.Diff(wantHeaders, reqHeaders); diff != "" { + m.t.Errorf("%s: mismatch challenge (-want +got):\n%s", m.name, diff) + } + } + + // check auth + if m.basic != "" { + header := req.Header.Get("Authorization") + if diff := cmp.Diff(header, fmt.Sprintf("Basic %s", m.basic)); diff != "" { + m.t.Errorf("%s: mismatch challenge (-want +got):\n%s", m.name, diff) + } + } + if m.bearer != "" { + header := req.Header.Get("Authorization") + if diff := cmp.Diff(header, fmt.Sprintf("Bearer %s", m.bearer)); diff != "" { + m.t.Errorf("%s: mismatch challenge (-want +got):\n%s", m.name, diff) + } + } + if m.apikey != "" { + header := req.Header.Get("api-key") + if diff := cmp.Diff(header, m.apikey); diff != "" { + m.t.Errorf("%s: mismatch challenge (-want +got):\n%s", m.name, diff) + } + } + return httpmock.NewStringResponse(m.status, ""), nil + +} + +func TestProducer(t *testing.T) { + fakeUrl := "https://jr.io" + + testCases := []struct { + name string + config phttp.Config + headers map[string]string + apiKey string + bearer string + basic string + status int + }{ + { + name: "test_simple_PUT", + config: phttp.Config{ + Endpoint: phttp.Endpoint{ + URL: fakeUrl, + Method: phttp.PUT, + }, + }, + status: http.StatusOK, + }, + { + name: "test_simple_POST", + config: phttp.Config{ + Endpoint: phttp.Endpoint{ + URL: fakeUrl, + Method: phttp.POST, + }, + }, + status: http.StatusOK, + }, + { + name: "test_ignore_status_code", + config: phttp.Config{ + Endpoint: phttp.Endpoint{ + URL: fakeUrl, + Method: phttp.POST, + }, + ErrorHandling: phttp.ErrorHandling{ + IgnoreStatusCode: true, + }, + }, + status: http.StatusTeapot, + }, + { + name: "test_expect_status_code", + config: phttp.Config{ + Endpoint: phttp.Endpoint{ + URL: fakeUrl, + Method: phttp.POST, + }, + ErrorHandling: phttp.ErrorHandling{ + ExpectStatusCode: http.StatusTeapot, + }, + }, + status: http.StatusTeapot, + }, + { + name: "test_with_headers", + config: phttp.Config{ + Endpoint: phttp.Endpoint{ + URL: fakeUrl, + Method: phttp.POST, + }, + Headers: map[string]string{ + "Test-Jrheader01": "value01", + "Test-Jrheader02": "value02", + }, + }, + status: http.StatusOK, + headers: map[string]string{ + "test-jrheader01": "value01", + "test-jrheader02": "value02", + }, + }, + { + name: "test_with_basic", + config: phttp.Config{ + Endpoint: phttp.Endpoint{ + URL: fakeUrl, + Method: phttp.POST, + }, + Authentication: phttp.Authentication{ + Type: phttp.BasicAuth, + Basic: phttp.Basic{ + Username: "user", + Password: "password", + }, + }, + }, + status: http.StatusOK, + basic: base64.StdEncoding.EncodeToString([]byte("user:password")), + }, + { + name: "test_with_bearer", + config: phttp.Config{ + Endpoint: phttp.Endpoint{ + URL: fakeUrl, + Method: phttp.POST, + }, + Authentication: phttp.Authentication{ + Type: phttp.BearerAuth, + Bearer: phttp.Bearer{ + Token: "sometoken", + }, + }, + }, + status: http.StatusOK, + bearer: "sometoken", + }, + { + name: "test_with_api_key", + config: phttp.Config{ + Endpoint: phttp.Endpoint{ + URL: fakeUrl, + Method: phttp.POST, + }, + Authentication: phttp.Authentication{ + Type: phttp.APIKeyAuth, + APIKey: phttp.APIKey{ + Header: "api-key", + Value: "jrapikey", + }, + }, + }, + status: http.StatusOK, + apiKey: "jrapikey", + }, + } + + for _, tc := range testCases { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + producer := phttp.Producer{} + producer.InitializeFromConfig(tc.config) + httpmock.ActivateNonDefault(producer.GetClient().GetClient()) + httpmock.Reset() + + mr := &mockResponder{ + name: tc.name, + t: t, + expectHeaders: tc.headers, + status: tc.status, + basic: tc.basic, + bearer: tc.bearer, + apikey: tc.apiKey, + } + httpmock.RegisterResponder(string(tc.config.Endpoint.Method), + fakeUrl, + mr.serveHTTP) + + producer.Produce([]byte("key"), []byte("{\"property\": \"value\"}"), nil) + httpmock.DeactivateAndReset() + }) + } + +} From 494c97cc191130088a128264c60c3c8b54006af1 Mon Sep 17 00:00:00 2001 From: Vincenzo Date: Fri, 16 Aug 2024 11:41:08 +0200 Subject: [PATCH 28/51] feat: add body check --- pkg/producers/http/producer_test.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/producers/http/producer_test.go b/pkg/producers/http/producer_test.go index f26545bf..0e6faef0 100644 --- a/pkg/producers/http/producer_test.go +++ b/pkg/producers/http/producer_test.go @@ -3,6 +3,7 @@ package http_test import ( "encoding/base64" "fmt" + "io" "net/http" "strings" "testing" @@ -12,6 +13,8 @@ import ( phttp "github.com/ugol/jr/pkg/producers/http" ) +var defaultBody = []byte("{\"property\": \"value\"}") + type mockResponder struct { name string t *testing.T @@ -24,6 +27,15 @@ type mockResponder struct { func (m *mockResponder) serveHTTP(req *http.Request) (*http.Response, error) { + body, err := io.ReadAll(req.Body) + defer req.Body.Close() + if err != nil { + m.t.Errorf("%s: cannot read request body", m.name) + } + if diff := cmp.Diff(defaultBody, body); diff != "" { + m.t.Errorf("%s: mismatch challenge (-want +got):\n%s", m.name, diff) + } + if m.expectHeaders != nil { // flatten request headers reqHeaders := make(map[string]string) @@ -220,7 +232,7 @@ func TestProducer(t *testing.T) { fakeUrl, mr.serveHTTP) - producer.Produce([]byte("key"), []byte("{\"property\": \"value\"}"), nil) + producer.Produce([]byte("key"), defaultBody, nil) httpmock.DeactivateAndReset() }) } From 532d69137a507467c0510f5aa7d414166db9b3f5 Mon Sep 17 00:00:00 2001 From: Vincenzo Date: Fri, 16 Aug 2024 12:38:57 +0200 Subject: [PATCH 29/51] fix: move tls certs to tls config --- pkg/producers/http/producer.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/producers/http/producer.go b/pkg/producers/http/producer.go index d2ebd1c2..dac93183 100644 --- a/pkg/producers/http/producer.go +++ b/pkg/producers/http/producer.go @@ -59,11 +59,13 @@ func (p *Producer) InitializeFromConfig(config Config) { log.Fatalf("KeyFile is set but CertFile is not") } + certificates := make([]tls.Certificate, 0) if p.configuration.TLS.CertFile != "" { p.certificate, err = tls.LoadX509KeyPair(p.configuration.TLS.CertFile, p.configuration.TLS.KeyFile) if err != nil { log.Fatalf("Failed to load certificate: %v", err) } + certificates = append(certificates, p.certificate) } if p.configuration.Session.UseCookieJar { @@ -77,6 +79,7 @@ func (p *Producer) InitializeFromConfig(config Config) { SetTimeout(p.configuration.Endpoint.timeout). SetTLSClientConfig(&tls.Config{ InsecureSkipVerify: p.configuration.TLS.InsecureSkipVerify, + Certificates: certificates, }). SetHeaders(p.configuration.Headers) @@ -84,10 +87,6 @@ func (p *Producer) InitializeFromConfig(config Config) { p.client.SetCookieJar(p.cookiejar) } - if p.configuration.TLS.CertFile != "" { - p.client.SetCertificates(p.certificate) - } - if p.configuration.TLS.RootCAFile != "" { p.client.SetRootCertificate(p.configuration.TLS.RootCAFile) } From ba9aca3440d42060210ed7e2855631a85ee37eae Mon Sep 17 00:00:00 2001 From: Vincenzo Date: Fri, 16 Aug 2024 12:42:50 +0200 Subject: [PATCH 30/51] fix: fix example --- pkg/producers/http/config.json.example | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/producers/http/config.json.example b/pkg/producers/http/config.json.example index a791df5e..081a8785 100644 --- a/pkg/producers/http/config.json.example +++ b/pkg/producers/http/config.json.example @@ -2,14 +2,14 @@ "endpoint": { "url": "https://jr.io", "method": "POST", - Timeout: "10s" + "timeout": "10s" }, "session":{ "use_cookie_jar": false }, "error_handling":{ - expect_status_code: 200, - ignore_status_code: false, + "expect_status_code": 200, + "ignore_status_code": false, }, "headers":{ "header01":"value01", @@ -19,7 +19,7 @@ "insecure_skip_verify": false, "cert_file": "/path/to/cert_file", "key_file": "/path/to/key_file", - "root_ca_file":/path/to/root_ca_file" + "root_ca_file":"/path/to/root_ca_file" }, "authentication":{ "type": "basic", From c6dad5fbcdbe471f26243ca440ea4a595a4ff7b8 Mon Sep 17 00:00:00 2001 From: ugol Date: Fri, 16 Aug 2024 13:15:57 +0200 Subject: [PATCH 31/51] minor fixes --- Readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Readme.md b/Readme.md index 18648074..d863745f 100644 --- a/Readme.md +++ b/Readme.md @@ -144,8 +144,8 @@ parse error: Expected value before ',' at line 1, column 5 ## Distributed Testing -JR can be run as a synthetic data generator for distributed testing. -At present the following testing tools are supported: +JR can be run as a distributed data generation. +At the moment the following testing tools are supported: - [k6](./k6/exec/) - [locust](./locust/) From 1a9ac6c63689a54f32ea300c9d25e5aa6eb22886 Mon Sep 17 00:00:00 2001 From: ugol Date: Fri, 16 Aug 2024 14:55:36 +0200 Subject: [PATCH 32/51] added XDG --- go.mod | 1 + go.sum | 2 ++ pkg/cmd/templateRun.go | 4 ++-- pkg/constants/constants.go | 11 +++++++++-- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 64f6a49e..849afb15 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.22.4 require ( cloud.google.com/go/storage v1.43.0 github.com/actgardner/gogen-avro/v10 v10.2.1 + github.com/adrg/xdg v0.5.0 github.com/aws/aws-sdk-go v1.54.14 github.com/confluentinc/confluent-kafka-go/v2 v2.5.0 github.com/elastic/go-elasticsearch/v8 v8.14.0 diff --git a/go.sum b/go.sum index 5c1149f4..961ad408 100644 --- a/go.sum +++ b/go.sum @@ -44,6 +44,8 @@ github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpH github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/actgardner/gogen-avro/v10 v10.2.1 h1:z3pOGblRjAJCYpkIJ8CmbMJdksi4rAhaygw0dyXZ930= github.com/actgardner/gogen-avro/v10 v10.2.1/go.mod h1:QUhjeHPchheYmMDni/Nx7VB0RsT/ee8YIgGY/xpEQgQ= +github.com/adrg/xdg v0.5.0 h1:dDaZvhMXatArP1NPHhnfaQUqWBLBsmx1h1HXQdMoFCY= +github.com/adrg/xdg v0.5.0/go.mod h1:dDdY4M4DF9Rjy4kHPeNL+ilVF+p2lK8IdM9/rTSGcI4= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= diff --git a/pkg/cmd/templateRun.go b/pkg/cmd/templateRun.go index 23c52144..598010db 100644 --- a/pkg/cmd/templateRun.go +++ b/pkg/cmd/templateRun.go @@ -118,7 +118,7 @@ jr template run --template "{{name}}" }) e := emitter.Emitter{ - Name: "cli", + Name: constants.DEFAULT_EMITTER_NAME, Locale: locale, Num: num, Frequency: frequency, @@ -136,7 +136,7 @@ jr template run --template "{{name}}" } functions.SetSeed(seed) - es := map[string][]emitter.Emitter{"cli": {e}} + es := map[string][]emitter.Emitter{constants.DEFAULT_EMITTER_NAME: {e}} RunEmitters([]string{e.Name}, es, false) }, } diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index da449057..709b6af0 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -20,13 +20,20 @@ package constants +import ( + "github.com/adrg/xdg" +) + var JRhome string const NUM = 1 const LOCALE = "us" const FREQUENCY = -1 const INFINITE = 1<<63 - 1 -const DEFAULT_HOMEDIR = "$HOME/.jr" + +// var DEFAULT_SYSTEMDIR = xdg.DataHome + "jr" +var DEFAULT_HOMEDIR = xdg.ConfigHome + ".jr" + const DEFAULT_KEY = "null" const DEFAULT_OUTPUT = "stdout" const DEFAULT_OUTPUT_TEMPLATE = "{{.V}}\n" @@ -36,7 +43,7 @@ const DEFAULT_PARTITIONS = 6 const DEFAULT_REPLICA = 3 const DEFAULT_PRELOAD_SIZE = 0 const DEFAULT_ENV_PREFIX = "JR" -const DEFAULT_EMITTER_NAME = "emitter" +const DEFAULT_EMITTER_NAME = "cli" const DEFAULT_VALUE_TEMPLATE = "user" const DEFAULT_TOPIC = "test" const DEFAULT_HTTP_PORT = 7482 From 05593b090c2d93568b472728c0cfc1e363465340 Mon Sep 17 00:00:00 2001 From: ugol Date: Fri, 16 Aug 2024 15:20:36 +0200 Subject: [PATCH 33/51] added XDG_HOME in makefile --- Makefile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index b3819cb4..52b5c239 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,10 @@ TIME=$(shell date) hello: @echo "JR,the JSON Random Generator" - + CONFIG_HOME=$XDG_CONFIG_HOME + ifeq (CONFIG_HOME, "" ) + HOME = "~" + endif install-gogen: go install github.com/actgardner/gogen-avro/v10/cmd/...@latest #go install github.com/hamba/avro/v2/cmd/avrogen@latest @@ -46,10 +49,10 @@ help: hello @echo '' copy_templates: - mkdir -p ~/.jr/kafka && cp -r templates ~/.jr/ && cp -r pkg/producers/kafka/*.properties.example ~/.jr/kafka/ + mkdir -p CONFIG_HOME/.jr/kafka && cp -r templates CONFIG_HOME/.jr/ && cp -r pkg/producers/kafka/*.properties.example CONFIG_HOME/.jr/kafka/ copy_config: - mkdir -p ~/.jr && cp config/* ~/.jr/ + mkdir -p CONFIG_HOME/.jr && cp config/* CONFIG_HOME/.jr/ install: install build/jr /usr/local/bin From 5988f039a91e5eb27cfff6b5baed5ec537ec0306 Mon Sep 17 00:00:00 2001 From: ugol Date: Fri, 16 Aug 2024 15:21:57 +0200 Subject: [PATCH 34/51] added XDG_HOME in makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 52b5c239..c0a6bfef 100644 --- a/Makefile +++ b/Makefile @@ -49,10 +49,10 @@ help: hello @echo '' copy_templates: - mkdir -p CONFIG_HOME/.jr/kafka && cp -r templates CONFIG_HOME/.jr/ && cp -r pkg/producers/kafka/*.properties.example CONFIG_HOME/.jr/kafka/ + mkdir -p $CONFIG_HOME/.jr/kafka && cp -r templates $CONFIG_HOME/.jr/ && cp -r pkg/producers/kafka/*.properties.example $CONFIG_HOME/.jr/kafka/ copy_config: - mkdir -p CONFIG_HOME/.jr && cp config/* CONFIG_HOME/.jr/ + mkdir -p $CONFIG_HOME/.jr && cp config/* $CONFIG_HOME/.jr/ install: install build/jr /usr/local/bin From 8b3e0808870fdc4fa6a06c228a61139bcd871fc5 Mon Sep 17 00:00:00 2001 From: ugol Date: Fri, 16 Aug 2024 15:43:28 +0200 Subject: [PATCH 35/51] added license in http producer --- pkg/producers/http/config.go | 20 ++++++++++++++++++++ pkg/producers/http/producer.go | 20 ++++++++++++++++++++ pkg/producers/http/producer_test.go | 20 ++++++++++++++++++++ 3 files changed, 60 insertions(+) diff --git a/pkg/producers/http/config.go b/pkg/producers/http/config.go index 20198e4a..2ad72430 100644 --- a/pkg/producers/http/config.go +++ b/pkg/producers/http/config.go @@ -1,3 +1,23 @@ +//Copyright © 2022 Vincenzo Marchese +// +//Permission is hereby granted, free of charge, to any person obtaining a copy +//of this software and associated documentation files (the "Software"), to deal +//in the Software without restriction, including without limitation the rights +//to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +//copies of the Software, and to permit persons to whom the Software is +//furnished to do so, subject to the following conditions: +// +//The above copyright notice and this permission notice shall be included in +//all copies or substantial portions of the Software. +// +//THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +//IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +//FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +//AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +//LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +//OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +//THE SOFTWARE. + package http import "time" diff --git a/pkg/producers/http/producer.go b/pkg/producers/http/producer.go index dac93183..9cdbdbe7 100644 --- a/pkg/producers/http/producer.go +++ b/pkg/producers/http/producer.go @@ -1,3 +1,23 @@ +//Copyright © 2022 Vincenzo Marchese +// +//Permission is hereby granted, free of charge, to any person obtaining a copy +//of this software and associated documentation files (the "Software"), to deal +//in the Software without restriction, including without limitation the rights +//to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +//copies of the Software, and to permit persons to whom the Software is +//furnished to do so, subject to the following conditions: +// +//The above copyright notice and this permission notice shall be included in +//all copies or substantial portions of the Software. +// +//THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +//IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +//FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +//AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +//LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +//OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +//THE SOFTWARE. + package http import ( diff --git a/pkg/producers/http/producer_test.go b/pkg/producers/http/producer_test.go index 0e6faef0..c5bc56c9 100644 --- a/pkg/producers/http/producer_test.go +++ b/pkg/producers/http/producer_test.go @@ -1,3 +1,23 @@ +//Copyright © 2022 Vincenzo Marchese +// +//Permission is hereby granted, free of charge, to any person obtaining a copy +//of this software and associated documentation files (the "Software"), to deal +//in the Software without restriction, including without limitation the rights +//to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +//copies of the Software, and to permit persons to whom the Software is +//furnished to do so, subject to the following conditions: +// +//The above copyright notice and this permission notice shall be included in +//all copies or substantial portions of the Software. +// +//THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +//IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +//FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +//AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +//LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +//OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +//THE SOFTWARE. + package http_test import ( From 43dda31c146fa21c2a8d9c24b37f11e69663412b Mon Sep 17 00:00:00 2001 From: ugol Date: Fri, 16 Aug 2024 16:18:29 +0200 Subject: [PATCH 36/51] minor fixes in makefile --- Makefile | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index c0a6bfef..a868ed5d 100644 --- a/Makefile +++ b/Makefile @@ -5,10 +5,12 @@ TIME=$(shell date) hello: @echo "JR,the JSON Random Generator" - CONFIG_HOME=$XDG_CONFIG_HOME - ifeq (CONFIG_HOME, "" ) - HOME = "~" + CONFIG_HOME=${XDG_CONFIG_HOME} + ifeq (${CONFIG_HOME}, "" ) + CONFIG_HOME=~ endif + CONFIG_HOME=~ + install-gogen: go install github.com/actgardner/gogen-avro/v10/cmd/...@latest #go install github.com/hamba/avro/v2/cmd/avrogen@latest @@ -49,10 +51,10 @@ help: hello @echo '' copy_templates: - mkdir -p $CONFIG_HOME/.jr/kafka && cp -r templates $CONFIG_HOME/.jr/ && cp -r pkg/producers/kafka/*.properties.example $CONFIG_HOME/.jr/kafka/ + mkdir -p ${CONFIG_HOME}/.jr/kafka && cp -r templates ${CONFIG_HOME}/.jr/ && cp -r pkg/producers/kafka/*.properties.example ${CONFIG_HOME}/.jr/kafka/ copy_config: - mkdir -p $CONFIG_HOME/.jr && cp config/* $CONFIG_HOME/.jr/ + mkdir -p ${CONFIG_HOME}/.jr && cp config/* ${CONFIG_HOME}/.jr/ install: install build/jr /usr/local/bin From 43adc1371170a0fdca40aaf150d0729c815c684c Mon Sep 17 00:00:00 2001 From: ugol Date: Fri, 16 Aug 2024 16:33:24 +0200 Subject: [PATCH 37/51] fixed typo --- pkg/functions/functionsDescription.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/functions/functionsDescription.go b/pkg/functions/functionsDescription.go index 52c422aa..2d4ad685 100644 --- a/pkg/functions/functionsDescription.go +++ b/pkg/functions/functionsDescription.go @@ -1258,7 +1258,7 @@ var funcDesc = map[string]FunctionDescription{ Name: "username", Category: "people", Description: "returns a random Username using Name, Surname", - Parameters: "name string, surname stringt", + Parameters: "name string, surname string", Localizable: true, Return: "string", Example: "jr template run --embedded '{{username \"barack\" \"obama\" }}'", From 66607f09e4876a08a283477ff0d28024fcc1e7ce Mon Sep 17 00:00:00 2001 From: Vincenzo Date: Fri, 16 Aug 2024 20:20:42 +0200 Subject: [PATCH 38/51] feat: substitute std log package with zerolog --- pkg/cmd/root.go | 17 +++--- pkg/cmd/server.go | 61 +++++++++----------- pkg/cmd/templateRun.go | 4 +- pkg/cmd/templateShow.go | 15 ++--- pkg/constants/constants.go | 5 +- pkg/emitter/emitter.go | 8 +-- pkg/functions/functions.go | 6 +- pkg/functions/helper.go | 9 +-- pkg/functions/people.go | 12 ++-- pkg/functions/timeAndDates.go | 7 ++- pkg/functions_test/templates_test.go | 28 ++++----- pkg/generator/generateRegistry.go | 15 ++--- pkg/producers/elastic/elasticProducer.go | 27 ++++----- pkg/producers/gcs/gcsProducer.go | 18 +++--- pkg/producers/http/producer.go | 20 +++---- pkg/producers/kafka/kafkaProducer.go | 72 +++++++++++++----------- pkg/producers/mongoDB/mongoProducer.go | 19 ++++--- pkg/producers/redis/redisProducer.go | 13 +++-- pkg/producers/s3/s3Producer.go | 17 +++--- pkg/producers/server/JsonProducer.go | 13 +++-- pkg/producers/test/TestProducer.go | 11 ++-- pkg/tpl/tpl.go | 5 +- 22 files changed, 211 insertions(+), 191 deletions(-) diff --git a/pkg/cmd/root.go b/pkg/cmd/root.go index 011b049e..5afa922e 100644 --- a/pkg/cmd/root.go +++ b/pkg/cmd/root.go @@ -22,16 +22,17 @@ package cmd import ( "fmt" + "os" + "strings" + "time" + + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/spf13/viper" "github.com/ugol/jr/pkg/configuration" "github.com/ugol/jr/pkg/constants" "github.com/ugol/jr/pkg/functions" - "log" - "os" - "strings" - "time" ) var rootCmd = &cobra.Command{ @@ -78,19 +79,19 @@ func initConfig() { viper.AddConfigPath(constants.JRhome) if err := viper.ReadInConfig(); err == nil { - log.Println("JR configuration:", viper.ConfigFileUsed()) + log.Debug().Str("file", viper.ConfigFileUsed()).Msg("JR configuration") } else { - log.Println("JR configuration not found") + log.Error().Err(err).Msg("JR configuration not found") } err := viper.UnmarshalKey("global", &configuration.GlobalCfg) if err != nil { - log.Println(err) + log.Error().Err(err).Msg("Failed to unmarshal global configuration") } //err = viper.UnmarshalKey("emitters", &emitters) err = viper.UnmarshalKey("emitters", &emitters2) if err != nil { - log.Println(err) + log.Error().Err(err).Msg("Failed to unmarshal emitter configuration") } seed := configuration.GlobalCfg.Seed if seed != -1 { diff --git a/pkg/cmd/server.go b/pkg/cmd/server.go index 2c79055e..8ff4e544 100644 --- a/pkg/cmd/server.go +++ b/pkg/cmd/server.go @@ -7,7 +7,6 @@ import ( "encoding/base64" "encoding/json" "fmt" - "log" "net/http" "path/filepath" "regexp" @@ -19,6 +18,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" "github.com/gorilla/sessions" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/ugol/jr/pkg/configuration" "github.com/ugol/jr/pkg/constants" @@ -79,7 +79,7 @@ var serverCmd = &cobra.Command{ port, err := cmd.Flags().GetInt("port") if err != nil { - log.Fatal(err) + log.Fatal().Err(err).Msg("Error getting port") } for i := 0; i < len(emitters); i++ { @@ -132,8 +132,8 @@ var serverCmd = &cobra.Command{ }) addr := fmt.Sprintf(":%d", port) - log.Printf("Starting HTTP server on port %d\n", port) - log.Fatal(http.ListenAndServe(addr, router)) + log.Info().Int("port", port).Msg("Starting HTTP server") + log.Fatal().Err(http.ListenAndServe(addr, router)) }, } @@ -242,7 +242,7 @@ func listEmitters(w http.ResponseWriter, r *http.Request) { _, err := w.Write([]byte(emitters_json)) if err != nil { - log.Println(err) + log.Error().Err(err).Msg("Error writing response") } } @@ -273,7 +273,7 @@ func addEmitter(w http.ResponseWriter, r *http.Request) { response := fmt.Sprintf("Emitter %s added", e.Name) _, err = w.Write([]byte(response)) if err != nil { - log.Println(err) + log.Error().Err(err).Msg("Error writing response") } } @@ -291,9 +291,8 @@ func startEmitter(w http.ResponseWriter, r *http.Request) { url := chi.URLParam(r, "emitter") _, err := w.Write([]byte("{\"started\":\"" + url + "\"}")) - if err != nil { - log.Println(err) + log.Error().Err(err).Msg("Error writing response") } } @@ -303,9 +302,8 @@ func stopEmitter(w http.ResponseWriter, r *http.Request) { url := chi.URLParam(r, "emitter") _, err := w.Write([]byte("{\"stopped\":\"" + url + "\"}")) - if err != nil { - log.Println(err) + log.Error().Err(err).Msg("Error writing response") } } @@ -315,9 +313,8 @@ func pauseEmitter(w http.ResponseWriter, r *http.Request) { url := chi.URLParam(r, "emitter") _, err := w.Write([]byte("{\"paused\":\"" + url + "\"}")) - if err != nil { - log.Println(err) + log.Error().Err(err).Msg("Error writing response") } } @@ -352,9 +349,8 @@ func statusEmitter(w http.ResponseWriter, r *http.Request) { url := chi.URLParam(r, "emitter") _, err := w.Write([]byte("{\"status\":\"" + url + "\"}")) - if err != nil { - log.Println(err) + log.Error().Err(err).Msg("Error writing response") } } @@ -379,7 +375,7 @@ func loadLastStatus(w http.ResponseWriter, r *http.Request) { _, err := w.Write(response.Bytes()) if err != nil { - log.Println(err) + log.Error().Err(err).Msg("Error writing response") } } @@ -389,7 +385,7 @@ func executeTemplate(w http.ResponseWriter, r *http.Request) { errorFormParse := r.ParseForm() if errorFormParse != nil { - log.Println("errorFormParse ", errorFormParse) + log.Error().Err(errorFormParse).Msg("Error parsing form") http.Error(w, errorFormParse.Error(), http.StatusInternalServerError) } @@ -402,9 +398,8 @@ func executeTemplate(w http.ResponseWriter, r *http.Request) { session.Save(r, w) templateParsed, errValidity := template.New("").Funcs(functions.FunctionsMap()).Parse(lastTemplateSubmittedValue) - if errValidity != nil { - log.Println("errValidity ", errValidity) + log.Error().Err(errValidity).Msg("Error parsing template") http.Error(w, errValidity.Error(), http.StatusInternalServerError) return } @@ -414,15 +409,14 @@ func executeTemplate(w http.ResponseWriter, r *http.Request) { errValidityRendering := templateParsed.Execute(&b, dummy) if errValidityRendering != nil { - log.Println("errValidityRendering = ", errValidityRendering) + log.Error().Err(errValidityRendering).Msg("Error rendering template") http.Error(w, errValidityRendering.Error(), http.StatusInternalServerError) return } _, err := w.Write([]byte(b.String())) - if err != nil { - log.Println(err) + log.Error().Err(err).Msg("Error writing response") } } @@ -445,36 +439,35 @@ func webPrintFunction(web_function_to_find string, w http.ResponseWriter, r *htt if len(matchingFunction) > 0 { + buffer := bytes.Buffer{} w.Header().Set("Content-Type", "application/json") - _, err := w.Write([]byte("{\"functions\":[")) + + buffer.WriteString("{\"functions\":[") for i, function_name := range matchingFunction { f, _ := functions.Description(function_name) b, errMarshal := json.Marshal(f) - if errMarshal != nil { - fmt.Println(errMarshal) + log.Error().Err(errMarshal).Msg("Error marshalling function") return } - if err != nil { - log.Println(err) - } - - _, err = w.Write(b) + buffer.Write(b) if i < len(matchingFunction)-1 { - _, err = w.Write([]byte(",")) + buffer.WriteString(",") } - if err != nil { - log.Println(err) - } } - _, err = w.Write([]byte("]}")) + buffer.WriteString("]}") + + _, err := w.Write(buffer.Bytes()) + if err != nil { + log.Error().Err(err).Msg("Error writing response") + } } else { http.Error(w, "No function found", http.StatusNotFound) diff --git a/pkg/cmd/templateRun.go b/pkg/cmd/templateRun.go index 598010db..60e83c53 100644 --- a/pkg/cmd/templateRun.go +++ b/pkg/cmd/templateRun.go @@ -21,9 +21,9 @@ package cmd import ( - "log" "time" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/ugol/jr/pkg/configuration" @@ -79,7 +79,7 @@ jr template run --template "{{name}}" throughput, err := emitter.ParseThroughput(throughputString) if err != nil { - log.Panicf("Throughput format error:%v", err) + log.Panic().Err(err).Msg("Throughput format error") } if throughput > 0 { diff --git a/pkg/cmd/templateShow.go b/pkg/cmd/templateShow.go index c8a08043..ba2d4c25 100644 --- a/pkg/cmd/templateShow.go +++ b/pkg/cmd/templateShow.go @@ -22,12 +22,13 @@ package cmd import ( "fmt" - "github.com/spf13/cobra" - "github.com/ugol/jr/pkg/constants" - "log" "os" "runtime" "strings" + + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + "github.com/ugol/jr/pkg/constants" ) var templateShowCmd = &cobra.Command{ @@ -37,7 +38,7 @@ var templateShowCmd = &cobra.Command{ Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { if len(args) == 0 { - log.Println("Template missing. Try the list command to see available templates") + log.Error().Msg("Template missing. Try the list command to see available templates") os.Exit(1) } @@ -46,11 +47,11 @@ var templateShowCmd = &cobra.Command{ templatePath := fmt.Sprintf("%s/%s.tpl", templateDir, args[0]) templateScript, err := os.ReadFile(templatePath) if err != nil { - log.Fatalf("Failed to ReadFile: %s", err) + log.Fatal().Err(err).Msg("Failed to ReadFile") } valid, err := isValidTemplate([]byte(templateScript)) if err != nil { - log.Fatalf("Failed to read a template: %s", err) + log.Fatal().Err(err).Msg("Failed to read a template") } templateString := string(templateScript) @@ -65,7 +66,7 @@ var templateShowCmd = &cobra.Command{ fmt.Println(templateString) fmt.Print(Reset) if !valid { - log.Println(err) + log.Fatal().Msg("Invalid template") } }, } diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 709b6af0..fddfb28e 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -21,6 +21,9 @@ package constants import ( + "fmt" + "os" + "github.com/adrg/xdg" ) @@ -32,7 +35,7 @@ const FREQUENCY = -1 const INFINITE = 1<<63 - 1 // var DEFAULT_SYSTEMDIR = xdg.DataHome + "jr" -var DEFAULT_HOMEDIR = xdg.ConfigHome + ".jr" +var DEFAULT_HOMEDIR = fmt.Sprintf("%s%c%s", xdg.ConfigHome, os.PathSeparator, "jr") const DEFAULT_KEY = "null" const DEFAULT_OUTPUT = "stdout" diff --git a/pkg/emitter/emitter.go b/pkg/emitter/emitter.go index c6cddfd8..da4f377b 100644 --- a/pkg/emitter/emitter.go +++ b/pkg/emitter/emitter.go @@ -22,10 +22,10 @@ package emitter import ( "fmt" - "log" "os" "time" + "github.com/rs/zerolog/log" "github.com/ugol/jr/pkg/configuration" "github.com/ugol/jr/pkg/constants" "github.com/ugol/jr/pkg/ctx" @@ -80,11 +80,11 @@ func (e *Emitter) Initialize(conf configuration.GlobalConfiguration) { keyTpl, err := tpl.NewTpl("key", e.KeyTemplate, functions.FunctionsMap(), &ctx.JrContext) if err != nil { - log.Fatal(err) + log.Fatal().Err(err).Msg("Failed to create key template") } valueTpl, err := tpl.NewTpl("value", e.EmbeddedTemplate, functions.FunctionsMap(), &ctx.JrContext) if err != nil { - log.Fatal(err) + log.Fatal().Err(err).Msg("Failed to create value template") } e.KTpl = keyTpl @@ -101,7 +101,7 @@ func (e *Emitter) Initialize(conf configuration.GlobalConfiguration) { return } else { if conf.SchemaRegistry { - log.Println("Ignoring schemaRegistry and/or serializer when output not set to kafka") + log.Warn().Msg("Ignoring schemaRegistry and/or serializer when output not set to kafka") } } diff --git a/pkg/functions/functions.go b/pkg/functions/functions.go index 68ea325c..ca4f12b4 100644 --- a/pkg/functions/functions.go +++ b/pkg/functions/functions.go @@ -24,7 +24,6 @@ import ( "bufio" "encoding/csv" "fmt" - "log" "math" "math/rand" "os" @@ -34,6 +33,7 @@ import ( "text/template" "github.com/google/uuid" + "github.com/rs/zerolog/log" "github.com/ugol/jr/pkg/constants" "github.com/ugol/jr/pkg/ctx" "golang.org/x/text/cases" @@ -449,12 +449,12 @@ func contains(values []int, value int) bool { func initialize(filename string) []string { file, err := os.Open(filename) if err != nil { - log.Printf("Failed to open file: %s", err) + log.Error().Err(err).Msg("Failed to open file") } defer func(file *os.File) { err := file.Close() if err != nil { - log.Printf("Error in closing file: %s", err) + log.Error().Err(err).Msg("Error in closing file") } }(file) diff --git a/pkg/functions/helper.go b/pkg/functions/helper.go index 3c8a58ac..a6c8f05e 100644 --- a/pkg/functions/helper.go +++ b/pkg/functions/helper.go @@ -22,10 +22,11 @@ package functions import ( "bytes" - "github.com/ugol/jr/pkg/ctx" - "log" "regexp" "text/template" + + "github.com/rs/zerolog/log" + "github.com/ugol/jr/pkg/ctx" ) func ExecuteTemplate(key *template.Template, value *template.Template, oneline bool) (string, string, error) { @@ -34,12 +35,12 @@ func ExecuteTemplate(key *template.Template, value *template.Template, oneline b var err error if err = key.Execute(&kBuffer, ctx.JrContext); err != nil { - log.Println(err) + log.Error().Err(err).Msg("Error executing key template") } k := kBuffer.String() if err = value.Execute(&vBuffer, ctx.JrContext); err != nil { - log.Println(err) + log.Error().Err(err).Msg("Error executing value template") } v := vBuffer.String() diff --git a/pkg/functions/people.go b/pkg/functions/people.go index ecd6eba2..3de230c3 100644 --- a/pkg/functions/people.go +++ b/pkg/functions/people.go @@ -22,11 +22,13 @@ package functions import ( "fmt" - "github.com/squeeze69/generacodicefiscale" - "github.com/ugol/jr/pkg/ctx" - "log" "strconv" "strings" + + "github.com/squeeze69/generacodicefiscale" + "github.com/ugol/jr/pkg/ctx" + + "github.com/rs/zerolog/log" ) // CodiceFiscale return a valid Italian Codice Fiscale @@ -66,11 +68,11 @@ func CodiceFiscale() string { codicecitta, erc := generacodicefiscale.CercaComune(city) if erc != nil { - log.Fatal(erc) + log.Fatal().Err(erc).Msg("Error in searching city") } cf, erg := generacodicefiscale.Genera(surname, name, gender, codicecitta.Codice, birthdate) if erg != nil { - log.Fatal(erg) + log.Fatal().Err(erg).Msg("Error in generating Codice Fiscale") } return cf } diff --git a/pkg/functions/timeAndDates.go b/pkg/functions/timeAndDates.go index af4deac5..ad59c2c8 100644 --- a/pkg/functions/timeAndDates.go +++ b/pkg/functions/timeAndDates.go @@ -21,8 +21,9 @@ package functions import ( - "log" "time" + + "github.com/rs/zerolog/log" ) // UnixTimeStamp returns a random unix timestamp not older than the given number of days @@ -38,12 +39,12 @@ func UnixTimeStamp(days int) int64 { func DateBetween(fromDate string, toDate string) string { start, err := time.Parse(time.DateOnly, fromDate) if err != nil { - log.Fatal(err) + log.Fatal().Err(err).Msg("Error parsing date") } end, err := time.Parse(time.DateOnly, toDate) if err != nil { - log.Fatal(err) + log.Fatal().Err(err).Msg("Error parsing date") } delta := end.Sub(start).Nanoseconds() diff --git a/pkg/functions_test/templates_test.go b/pkg/functions_test/templates_test.go index ce11d407..2602f893 100644 --- a/pkg/functions_test/templates_test.go +++ b/pkg/functions_test/templates_test.go @@ -22,11 +22,11 @@ package functions_test import ( "bytes" - "github.com/ugol/jr/pkg/functions" - "log" "strconv" "testing" "text/template" + + "github.com/ugol/jr/pkg/functions" ) func TestSimpleContext(t *testing.T) { @@ -196,11 +196,11 @@ func Test2TemplatesWithCommonId(t *testing.T) { user, err := v.New("user").Parse(userTemplate) if err != nil { - log.Fatal(err) + t.Fatal(err) } order, err := v.New("order").Parse(orderTemplate) if err != nil { - log.Fatal(err) + t.Fatal(err) } var expectUser bytes.Buffer @@ -231,15 +231,15 @@ func Test2TemplatesWithValueFromList(t *testing.T) { user, err := v.New("user").Parse(userTemplate) if err != nil { - log.Fatal(err) + t.Fatal(err) } user2, err := v.New("user").Parse(userTemplate2) if err != nil { - log.Fatal(err) + t.Fatal(err) } order, err := v.New("order").Parse(orderTemplate) if err != nil { - log.Fatal(err) + t.Fatal(err) } var expectUser bytes.Buffer @@ -277,7 +277,7 @@ func TestTemplatesWithValueFromListAtIndex(t *testing.T) { tOne, err := v.New("user").Parse(templateOne) if err != nil { - log.Fatal(err) + t.Fatal(err) } var expectedOne bytes.Buffer @@ -285,7 +285,7 @@ func TestTemplatesWithValueFromListAtIndex(t *testing.T) { tTwo, err := v.New("user").Parse(templateTwo) if err != nil { - log.Fatal(err) + t.Fatal(err) } var expectedTwo bytes.Buffer @@ -293,7 +293,7 @@ func TestTemplatesWithValueFromListAtIndex(t *testing.T) { check, err := v.New("order").Parse(checkTemplate) if err != nil { - log.Fatal(err) + t.Fatal(err) } var expectCheck bytes.Buffer @@ -317,7 +317,7 @@ func TestTemplatesWithValueFromListAtIndex_greater_than_length(t *testing.T) { tOne, err := v.New("user").Parse(templateOne) if err != nil { - log.Fatal(err) + t.Fatal(err) } var expectedOne bytes.Buffer @@ -325,7 +325,7 @@ func TestTemplatesWithValueFromListAtIndex_greater_than_length(t *testing.T) { tTwo, err := v.New("user").Parse(templateTwo) if err != nil { - log.Fatal(err) + t.Fatal(err) } var expectedTwo bytes.Buffer @@ -333,7 +333,7 @@ func TestTemplatesWithValueFromListAtIndex_greater_than_length(t *testing.T) { check, err := v.New("order").Parse(checkTemplate) if err != nil { - log.Fatal(err) + t.Fatal(err) } var expectCheck bytes.Buffer @@ -360,7 +360,7 @@ func TestManyTemplates(t *testing.T) { for i := 0; i < len(v); i++ { _, err := tpl.New(strconv.Itoa(i)).Parse((v[i])) if err != nil { - log.Fatal(err) + t.Fatal(err) } } diff --git a/pkg/generator/generateRegistry.go b/pkg/generator/generateRegistry.go index 9a2de7ec..ed25017c 100644 --- a/pkg/generator/generateRegistry.go +++ b/pkg/generator/generateRegistry.go @@ -5,13 +5,14 @@ package main import ( "bytes" "go/format" - "golang.org/x/text/cases" - "golang.org/x/text/language" - "log" "os" "path/filepath" "strings" "text/template" + + "github.com/rs/zerolog/log" + "golang.org/x/text/cases" + "golang.org/x/text/language" ) func main() { @@ -79,23 +80,23 @@ func main() { var b bytes.Buffer err = t.Execute(&b, d) if err != nil { - log.Fatal(err) + log.Fatal().Err(err).Msg("Error executing template") } bb, err := format.Source(b.Bytes()) if err != nil { - log.Fatal(err) + log.Fatal().Err(err).Msg("Error formatting source") } initFile, err := os.OpenFile("../types/registry.go", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755) if err != nil { - log.Fatal(err) + log.Fatal().Err(err).Msg("Error opening file") } defer initFile.Close() _, err = initFile.WriteString(string(bb)) if err != nil { - log.Fatal(err) + log.Fatal().Err(err).Msg("Error writing to init file") } } diff --git a/pkg/producers/elastic/elasticProducer.go b/pkg/producers/elastic/elasticProducer.go index b074d0a6..6dd3cceb 100644 --- a/pkg/producers/elastic/elasticProducer.go +++ b/pkg/producers/elastic/elasticProducer.go @@ -4,15 +4,16 @@ import ( "context" "crypto/tls" "encoding/json" - "github.com/elastic/go-elasticsearch/v8" - "github.com/elastic/go-elasticsearch/v8/esapi" - "github.com/google/uuid" - "io/ioutil" - "log" "net" "net/http" + "os" "strings" "time" + + "github.com/elastic/go-elasticsearch/v8" + "github.com/elastic/go-elasticsearch/v8/esapi" + "github.com/google/uuid" + "github.com/rs/zerolog/log" ) type Config struct { @@ -29,16 +30,16 @@ type ElasticProducer struct { func (p *ElasticProducer) Initialize(configFile string) { var config Config - file, err := ioutil.ReadFile(configFile) + file, err := os.ReadFile(configFile) if err != nil { - log.Fatalf("Failed to read configuration file: %s", err) + log.Fatal().Err(err).Msg("Failed to read configuration file") } err = json.Unmarshal(file, &config) if err != nil { - log.Fatalf("Failed to ReadFile: %s", err) + log.Fatal().Err(err).Msg("Failed to ReadFile") } if err != nil { - log.Fatalf("Failed to parse configuration parameters: %s", err) + log.Fatal().Err(err).Msg("Failed to parse configuration parameters") } cfg := elasticsearch.Config{ @@ -58,7 +59,7 @@ func (p *ElasticProducer) Initialize(configFile string) { client, err := elasticsearch.NewClient(cfg) if err != nil { - log.Fatalf("Can't connect to Elastic: %s", err) + log.Fatal().Err(err).Msg("Can't connect to Elastic") } p.index = config.ElasticIndex @@ -90,16 +91,16 @@ func (p *ElasticProducer) Produce(k []byte, v []byte, o any) { res, err := req.Do(context.Background(), &p.client) if err != nil { - log.Fatalf("Failed to write data in Elastic:\n%s", err) + log.Fatal().Err(err).Msg("Failed to write data in Elastic") } defer res.Body.Close() if res.IsError() { - log.Fatalf("failed to index document: %s", res.String()) + log.Fatal().Str("response", res.String()).Msg("failed to index document") } } func (p *ElasticProducer) Close() error { - log.Println("elasticsearch Client doesn't provide a close method!") + log.Warn().Msg("elasticsearch Client doesn't provide a close method!") return nil } diff --git a/pkg/producers/gcs/gcsProducer.go b/pkg/producers/gcs/gcsProducer.go index 9494d547..392d3d70 100644 --- a/pkg/producers/gcs/gcsProducer.go +++ b/pkg/producers/gcs/gcsProducer.go @@ -1,13 +1,15 @@ package gcs import ( - "cloud.google.com/go/storage" "context" "encoding/json" "fmt" + "os" + + "cloud.google.com/go/storage" "github.com/google/uuid" - "io/ioutil" - "log" + + "github.com/rs/zerolog/log" ) type Config struct { @@ -21,14 +23,14 @@ type GCSProducer struct { func (p *GCSProducer) Initialize(configFile string) { var config Config - file, err := ioutil.ReadFile(configFile) + file, err := os.ReadFile(configFile) if err != nil { - log.Fatalf("Failed to read configuration file: %s", err) + log.Fatal().Err(err).Msg("Failed to read configuration file") } err = json.Unmarshal(file, &config) if err != nil { - log.Fatalf("Failed to parse configuration parameters: %s", err) + log.Fatal().Err(err).Msg("Failed to parse configuration parameters") } ctx := context.Background() @@ -37,7 +39,7 @@ func (p *GCSProducer) Initialize(configFile string) { // https://developers.google.com/identity/protocols/application-default-credentials. client, err := storage.NewClient(ctx) if err != nil { - log.Fatalf("Failed to create client: %v", err) + log.Fatal().Err(err).Msg("Failed to create client") } p.client = *client @@ -64,7 +66,7 @@ func (p *GCSProducer) Produce(k []byte, v []byte, o any) { _, err := writer.Write([]byte(kvPair)) if err != nil { - log.Fatalf("Failed to write to GCS: %v", err) + log.Fatal().Err(err).Msg("Failed to write to GCS") } writer.Close() diff --git a/pkg/producers/http/producer.go b/pkg/producers/http/producer.go index dac93183..d77e296f 100644 --- a/pkg/producers/http/producer.go +++ b/pkg/producers/http/producer.go @@ -3,13 +3,13 @@ package http import ( "crypto/tls" "encoding/json" - "log" "net/http" "net/http/cookiejar" "os" "time" "github.com/go-resty/resty/v2" + "github.com/rs/zerolog/log" ) type Producer struct { @@ -23,12 +23,12 @@ type Producer struct { func (p *Producer) Initialize(configFile string) { cfgBytes, err := os.ReadFile(configFile) if err != nil { - log.Fatalf("Failed to read config file: %v", err) + log.Fatal().Err(err).Msg("Failed to read config file") } config := Config{} if err := json.Unmarshal(cfgBytes, &config); err != nil { - log.Fatalf("Failed to unmarshal config: %v", err) + log.Fatal().Err(err).Msg("Failed to unmarshal config") } p.InitializeFromConfig(config) @@ -43,7 +43,7 @@ func (p *Producer) InitializeFromConfig(config Config) { } else { p.configuration.Endpoint.timeout, err = time.ParseDuration(p.configuration.Endpoint.Timeout) if err != nil { - log.Fatalf("Failed to parse timeout: %v", err) + log.Fatal().Err(err).Msg("Failed to parse timeout") } } @@ -53,17 +53,17 @@ func (p *Producer) InitializeFromConfig(config Config) { } if p.configuration.TLS.CertFile != "" && p.configuration.TLS.KeyFile == "" { - log.Fatalf("CertFile is set but KeyFile is not") + log.Fatal().Err(err).Msg("CertFile is set but KeyFile is not") } if p.configuration.TLS.CertFile == "" && p.configuration.TLS.KeyFile != "" { - log.Fatalf("KeyFile is set but CertFile is not") + log.Fatal().Err(err).Msg("KeyFile is set but CertFile is not") } certificates := make([]tls.Certificate, 0) if p.configuration.TLS.CertFile != "" { p.certificate, err = tls.LoadX509KeyPair(p.configuration.TLS.CertFile, p.configuration.TLS.KeyFile) if err != nil { - log.Fatalf("Failed to load certificate: %v", err) + log.Fatal().Err(err).Msg("Failed to load certificate") } certificates = append(certificates, p.certificate) } @@ -71,7 +71,7 @@ func (p *Producer) InitializeFromConfig(config Config) { if p.configuration.Session.UseCookieJar { p.cookiejar, err = cookiejar.New(nil) if err != nil { - log.Fatalf("Failed to create cookie jar: %v", err) + log.Fatal().Err(err).Msg("Failed to create cookie jar") } } @@ -132,12 +132,12 @@ func (p *Producer) Produce(k []byte, v []byte, o any) { } if err != nil { - log.Fatalf("Failed to send request: %v", err) + log.Fatal().Err(err).Msg("Failed to send request") } if resp.StatusCode() != p.configuration.ErrorHandling.ExpectStatusCode && !p.configuration.ErrorHandling.IgnoreStatusCode { - log.Fatalf("Unexpected status code: %d", resp.StatusCode()) + log.Fatal().Int("statusCode", resp.StatusCode()).Msg("Unexpected status code") } } diff --git a/pkg/producers/kafka/kafkaProducer.go b/pkg/producers/kafka/kafkaProducer.go index 49959e24..e73a4bd1 100644 --- a/pkg/producers/kafka/kafkaProducer.go +++ b/pkg/producers/kafka/kafkaProducer.go @@ -25,6 +25,15 @@ import ( "context" "encoding/json" "fmt" + "io" + "os" + "path/filepath" + "regexp" + "runtime" + "strconv" + "strings" + "time" + "github.com/confluentinc/confluent-kafka-go/v2/kafka" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rules/encryption" @@ -36,15 +45,8 @@ import ( "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde/avrov2" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde/jsonschema" "github.com/ugol/jr/pkg/types" - "io/ioutil" - "log" - "os" - "path/filepath" - "regexp" - "runtime" - "strconv" - "strings" - "time" + + "github.com/rs/zerolog/log" ) type KafkaManager struct { @@ -64,11 +66,11 @@ func (k *KafkaManager) Initialize(configFile string) { conf := convertInKafkaConfig(readConfig(configFile)) k.admin, err = kafka.NewAdminClient(&conf) if err != nil { - log.Fatalf("Failed to create admin client: %s", err) + log.Fatal().Err(err).Msg("Failed to create admin client") } k.producer, err = kafka.NewProducer(&conf) if err != nil { - log.Fatalf("Failed to create producer: %s", err) + log.Fatal().Err(err).Msg("Failed to create producer") } } @@ -82,7 +84,7 @@ func (k *KafkaManager) InitializeSchemaRegistry(configFile string) { conf["schemaRegistryPassword"])) if err != nil { - log.Fatalf("Failed to create schema registry client: %s", err) + log.Fatal().Err(err).Msg("Failed to create schema registry client") } if k.Serializer == "avro" || k.Serializer == "avro-generic" { @@ -102,13 +104,13 @@ func verifyCSFLE(conf map[string]string, k *KafkaManager) { filePath := filepath.Join(currentDir, "../../types/"+k.TemplateType+".avsc") file, err := os.Open(filePath) if err != nil { - log.Fatalf("Failed to open file: %s", err) + log.Fatal().Err(err).Msg("Failed to open file") } defer file.Close() - content, err := ioutil.ReadAll(file) + content, err := io.ReadAll(file) if err != nil { - log.Fatalf("Failed to read file: %s", err) + log.Fatal().Err(err).Msg("Failed to read file") } contentString := string(content) @@ -158,8 +160,7 @@ func verifyCSFLE(conf map[string]string, k *KafkaManager) { } _, err = k.schema.Register(k.Topic+"-value", schema, true) if err != nil { - fmt.Printf("Failed to register schema: %s\n", err) - os.Exit(1) + log.Fatal().Err(err).Msg("Failed to register schema") } k.fleEnabled = true @@ -203,26 +204,26 @@ func (k *KafkaManager) Produce(key []byte, data []byte, o any) { ser, err = avrov2.NewSerializer(k.schema, serde.ValueSerde, serConfig) } else if k.Serializer == "protobuf" { //ser, err = protobuf.NewSerializer(k.schema, serde.ValueSerde, protobuf.NewSerializerConfig()) - log.Fatal("Protobuf not yet implemented") + log.Fatal().Msg("Protobuf not yet implemented") } else if k.Serializer == "json-schema" { ser, err = jsonschema.NewSerializer(k.schema, serde.ValueSerde, jsonschema.NewSerializerConfig()) } else { - log.Fatalf("Serializer '%v' not supported", k.Serializer) + log.Fatal().Str("serializer", k.Serializer).Msg("Serializer not supported") } if err != nil { - log.Fatalf("Error creating serializer: %s\n", err) + log.Fatal().Err(err).Msg("Error creating serializer") } else { t := types.GetType(k.TemplateType) err := json.Unmarshal(data, &t) if err != nil { - log.Fatalf("Failed to unmarshal data: %s\n", err) + log.Fatal().Err(err).Msg("Failed to unmarshal data") } payload, err := ser.Serialize(k.Topic, t) if err != nil { - log.Fatalf("Failed to serialize payload: %s\n", err) + log.Fatal().Err(err).Msg("Failed to serialize payload") } else { data = payload } @@ -248,7 +249,7 @@ func (k *KafkaManager) Produce(key []byte, data []byte, o any) { //time.Sleep(time.Second) //continue } - log.Printf("Failed to produce message: %v\n", err) + log.Error().Err(err).Msg("Failed to produce message") } } @@ -276,14 +277,18 @@ func (k *KafkaManager) CreateTopicFull(topic string, partitions int, rf int) { kafka.SetAdminOperationTimeout(maxDuration)) if err != nil { - log.Printf("Problem during the topic creation: %v\n", err) + log.Error().Err(err).Msg("Problem during the topic creation") } // Check for specific topic errors for _, result := range results { if result.Error.Code() != kafka.ErrNoError && result.Error.Code() != kafka.ErrTopicAlreadyExists { - log.Fatalf("Topic creation failed for %s: %v", result.Topic, result.Error.String()) + log.Fatal(). + Str("topic", result.Topic). + Str("code", result.Error.Code().String()). + Err(fmt.Errorf(result.Error.String())). + Msg("Topic creation failed") } } @@ -298,12 +303,12 @@ func listenToEventsFrom(k *kafka.Producer, topic string) { case *kafka.Message: m := ev if m.TopicPartition.Error != nil { - log.Printf("Delivery failed: %v\n", m.TopicPartition.Error) + log.Error().Err(m.TopicPartition.Error).Msg("Delivery failed") } else { //fmt.Printf("Delivered message to topic %s [%d] at offset %v\n", *m.TopicPartition.Topic, m.TopicPartition.Partition, m.TopicPartition.Offset) } case kafka.Error: - log.Printf("Error: %v\n", ev) + log.Error().Err(ev).Msg("Kafka error") case *kafka.Stats: // https://github.com/confluentinc/librdkafka/blob/master/STATISTICS.md var stats map[string]interface{} @@ -315,10 +320,13 @@ func listenToEventsFrom(k *kafka.Producer, topic string) { b, _ := strconv.Atoi(strings.TrimSpace(txbytes)) if b > 0 { - log.Printf("%s bytes produced to topic %s \n", txbytes, topic) + log.Info(). + Str("bytes", txbytes). + Str("topic", topic). + Msg("Bytes produced to topic") } default: - log.Printf("Ignored event: %s\n", ev) + log.Warn().Interface("ev", ev).Msg("Ignored event") } } @@ -330,12 +338,12 @@ func readConfig(configFile string) map[string]string { file, err := os.Open(configFile) if err != nil { - log.Fatalf("Failed to open file: %s", err) + log.Fatal().Err(err).Msg("Failed to open configuration file") } defer func(file *os.File) { err := file.Close() if err != nil { - log.Fatalf("Error in closing file: %s", err) + log.Fatal().Err(err).Msg("Error in closing file") } }(file) @@ -351,7 +359,7 @@ func readConfig(configFile string) map[string]string { } if err := scanner.Err(); err != nil { - log.Fatalf("Failed to read file: %s", err) + log.Fatal().Err(err).Msg("Failed to read file") } return m } diff --git a/pkg/producers/mongoDB/mongoProducer.go b/pkg/producers/mongoDB/mongoProducer.go index 4d5ee58d..a0f4c8d6 100644 --- a/pkg/producers/mongoDB/mongoProducer.go +++ b/pkg/producers/mongoDB/mongoProducer.go @@ -3,10 +3,11 @@ package mongoDB import ( "context" "encoding/json" + "os" + + "github.com/rs/zerolog/log" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" - "io/ioutil" - "log" ) type Config struct { @@ -25,13 +26,13 @@ type MongoProducer struct { func (p *MongoProducer) Initialize(configFile string) { var config Config - file, err := ioutil.ReadFile(configFile) + file, err := os.ReadFile(configFile) if err != nil { - log.Fatalf("Failed to ReadFile: %s", err) + log.Fatal().Err(err).Msg("Failed to read configuration file") } err = json.Unmarshal(file, &config) if err != nil { - log.Fatalf("Failed to parse configuration parameters: %s", err) + log.Fatal().Err(err).Msg("Failed to parse configuration parameters") } clientOptions := options.Client().ApplyURI(config.MongoURI) @@ -48,7 +49,7 @@ func (p *MongoProducer) Initialize(configFile string) { client, err := mongo.Connect(context.Background(), clientOptions) if err != nil { - log.Fatalf("Can't connect to Mongo: %s", err) + log.Fatal().Err(err).Msg("Can't connect to Mongo") } p.client = *client @@ -61,7 +62,7 @@ func (p *MongoProducer) Produce(k []byte, v []byte, o any) { var dev map[string]interface{} err := json.Unmarshal(v, &dev) if err != nil { - log.Fatalf("Failed to unmarshal json:\n%s", err) + log.Fatal().Err(err).Msg("Failed to unmarshal json") } if k == nil || len(k) == 0 { @@ -70,14 +71,14 @@ func (p *MongoProducer) Produce(k []byte, v []byte, o any) { _, err = collection.InsertOne(context.Background(), dev) if err != nil { - log.Fatalf("Failed to write data in Mongo:\n%s", err) + log.Fatal().Err(err).Msg("Failed to write data in Mongo") } } func (p *MongoProducer) Close() error { err := p.client.Disconnect(context.Background()) if err != nil { - log.Printf("Failed to close Mongo connection:\n%s", err) + log.Warn().Err(err).Msg("Failed to close Mongo connection") } return err } diff --git a/pkg/producers/redis/redisProducer.go b/pkg/producers/redis/redisProducer.go index 66ffe4f1..d415bcef 100644 --- a/pkg/producers/redis/redisProducer.go +++ b/pkg/producers/redis/redisProducer.go @@ -3,10 +3,11 @@ package redis import ( "context" "encoding/json" - "github.com/redis/go-redis/v9" - "log" "os" "time" + + "github.com/redis/go-redis/v9" + "github.com/rs/zerolog/log" ) type RedisProducer struct { @@ -19,12 +20,12 @@ func (p *RedisProducer) Initialize(configFile string) { data, err := os.ReadFile(configFile) if err != nil { - log.Fatalf("Failed to load Redis configFile: %s", err) + log.Fatal().Err(err).Msg("Failed to load Redis configFile") } err = json.Unmarshal(data, &options) if err != nil { - log.Fatalf("Failed to parsa configuration parameters: %s", err) + log.Fatal().Err(err).Msg("Failed to parse configuration parameters") } p.client = *redis.NewClient(&options) @@ -33,7 +34,7 @@ func (p *RedisProducer) Initialize(configFile string) { func (p *RedisProducer) Close() error { err := p.client.Close() if err != nil { - log.Printf("Failed to close Redis connection:\n%s", err) + log.Warn().Err(err).Msg("Failed to close Redis connection") } return err } @@ -42,6 +43,6 @@ func (p *RedisProducer) Produce(k []byte, v []byte, o any) { ctx := context.Background() err := p.client.Set(ctx, string(k), string(v), p.Ttl).Err() if err != nil { - log.Fatalf("Failed to write data in Redis:\n%s", err) + log.Fatal().Err(err).Msg("Failed to write data in Redis") } } diff --git a/pkg/producers/s3/s3Producer.go b/pkg/producers/s3/s3Producer.go index 381382f7..0bee6e68 100644 --- a/pkg/producers/s3/s3Producer.go +++ b/pkg/producers/s3/s3Producer.go @@ -3,12 +3,13 @@ package s3 import ( "bytes" "encoding/json" + "os" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/google/uuid" - "io/ioutil" - "log" + "github.com/rs/zerolog/log" ) type Config struct { @@ -23,19 +24,19 @@ type S3Producer struct { func (p *S3Producer) Initialize(configFile string) { var config Config - file, err := ioutil.ReadFile(configFile) + file, err := os.ReadFile(configFile) if err != nil { - log.Fatalf("Failed to ReadFile: %s", err) + log.Fatal().Err(err).Msg("Failed to ReadFile") } err = json.Unmarshal(file, &config) if err != nil { - log.Fatalf("Failed to parse configuration parameters: %s", err) + log.Fatal().Err(err).Msg("Failed to parse configuration parameters") } sess, err := session.NewSession(&aws.Config{Region: &config.AWSRegion}) if err != nil { - log.Fatalf("Can't establish a session to S3: %s", err) + log.Fatal().Err(err).Msg("Can't establish a session to S3") return } @@ -67,11 +68,11 @@ func (p *S3Producer) Produce(k []byte, v []byte, o any) { }) if err != nil { - log.Fatalf("Failed to write data in s3:\n%s", err) + log.Fatal().Err(err).Msg("Failed to write data in s3") } } func (p *S3Producer) Close() error { - log.Println("S3 Client doesn't provide a close method!") + log.Warn().Msg("S3 Client doesn't provide a close method!") return nil } diff --git a/pkg/producers/server/JsonProducer.go b/pkg/producers/server/JsonProducer.go index 30350699..03ae3287 100644 --- a/pkg/producers/server/JsonProducer.go +++ b/pkg/producers/server/JsonProducer.go @@ -1,9 +1,10 @@ package server import ( - "github.com/ugol/jr/pkg/tpl" - "log" "net/http" + + "github.com/rs/zerolog/log" + "github.com/ugol/jr/pkg/tpl" ) type JsonProducer struct { @@ -22,19 +23,19 @@ func (c *JsonProducer) Produce(key []byte, value []byte, o interface{}) { if string(key) != "null" { _, err := (respWriter).Write(key) if err != nil { - log.Println(err.Error()) + log.Error().Err(err).Msg("Error writing key") } _, err = (respWriter).Write([]byte(",")) if err != nil { - log.Println(err.Error()) + log.Error().Err(err).Msg("Error writing comma") } } _, err := (respWriter).Write(value) if err != nil { - log.Println(err.Error()) + log.Error().Err(err).Msg("Error writing value") } } else { - log.Printf("Server producer must produce to a http.ResponseWriter, but was a %T\n", o) + log.Warn().Interface("o", o).Msg("Server producer must produce to a http.ResponseWriter") } } diff --git a/pkg/producers/test/TestProducer.go b/pkg/producers/test/TestProducer.go index f265bf5d..47a41389 100644 --- a/pkg/producers/test/TestProducer.go +++ b/pkg/producers/test/TestProducer.go @@ -2,8 +2,9 @@ package test import ( "bytes" + + "github.com/rs/zerolog/log" "github.com/ugol/jr/pkg/tpl" - "log" ) type TestProducer struct { @@ -22,19 +23,19 @@ func (c *TestProducer) Produce(key []byte, value []byte, o interface{}) { if string(key) != "null" { _, err := (respWriter).Write(key) if err != nil { - log.Println(err.Error()) + log.Error().Err(err).Msg("Error writing key") } _, err = (respWriter).Write([]byte(",")) if err != nil { - log.Println(err.Error()) + log.Error().Err(err).Msg("Error writing comma") } } _, err := (respWriter).Write(value) if err != nil { - log.Println(err.Error()) + log.Error().Err(err).Msg("Error writing value") } } else { - log.Printf("Test producer must produce to a bytes.Buffer, but was a %T\n", o) + log.Warn().Interface("o", o).Msg("Test producer must produce to a bytes.Buffer") } } diff --git a/pkg/tpl/tpl.go b/pkg/tpl/tpl.go index b6ad992b..4c6d8f1a 100644 --- a/pkg/tpl/tpl.go +++ b/pkg/tpl/tpl.go @@ -22,8 +22,9 @@ package tpl import ( "bytes" - "log" "text/template" + + "github.com/rs/zerolog/log" ) type Tpl struct { @@ -50,7 +51,7 @@ func (t *Tpl) ExecuteWith(data any) string { var buffer bytes.Buffer err := t.Template.Execute(&buffer, data) if err != nil { - log.Fatal(err) + log.Fatal().Err(err).Msg("Error executing template") } return buffer.String() } From dbfa6ba65cd49f568feb211bc4f67350e919abfb Mon Sep 17 00:00:00 2001 From: Vincenzo Date: Fri, 16 Aug 2024 20:20:50 +0200 Subject: [PATCH 39/51] feat: substitute std log package with zerolog --- go.mod | 3 +++ go.sum | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/go.mod b/go.mod index 849afb15..776ef0a0 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/gorilla/sessions v1.3.0 github.com/jarcoal/httpmock v1.3.1 github.com/redis/go-redis/v9 v9.5.3 + github.com/rs/zerolog v1.33.0 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 github.com/squeeze69/generacodicefiscale v1.0.5 @@ -80,6 +81,8 @@ require ( github.com/klauspost/compress v1.17.7 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect diff --git a/go.sum b/go.sum index 961ad408..2ae99145 100644 --- a/go.sum +++ b/go.sum @@ -117,6 +117,7 @@ github.com/containerd/ttrpc v1.2.3 h1:4jlhbXIGvijRtNC8F/5CpuJZ7yKOBFGFOOXg1bkISz github.com/containerd/ttrpc v1.2.3/go.mod h1:ieWsXucbb8Mj9PH0rXCw1i8IunRbbAiDkpXkbfflWBM= github.com/containerd/typeurl/v2 v2.1.1 h1:3Q4Pt7i8nYwy2KmQWIw2+1hTvwTE/6w9FqcttATPO/4= github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -198,6 +199,7 @@ github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0= @@ -335,6 +337,8 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= @@ -430,6 +434,9 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= +github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= @@ -632,8 +639,10 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= From 90805a2e7c99e32b320163ed8d330bb7c0b30c18 Mon Sep 17 00:00:00 2001 From: Vincenzo Date: Fri, 16 Aug 2024 20:21:13 +0200 Subject: [PATCH 40/51] fix: improved OS recognition --- Makefile | 47 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index c0a6bfef..a13211d2 100644 --- a/Makefile +++ b/Makefile @@ -2,13 +2,37 @@ VERSION=0.3.9 GOVERSION=$(shell go version) USER=$(shell id -u -n) TIME=$(shell date) +JR_HOME=jr + +ifndef XDG_CONFIG_HOME +ifeq ($(OS), Windows_NT) + detectedOS := Windows +else + detectedOS := $(shell sh -c 'uname 2>/dev/null || echo Unknown') +endif + +ifeq ($(detectedOS), Darwin) + CONFIG_HOME="$(HOME)/Library/Application Support" +endif +ifeq ($(detectedOS), Linux) + CONFIG_HOME="$(HOME)/.config" +endif +ifeq ($(detectedOS), Windows_NT) + CONFIG_HOME="$(LOCALAPPDATA)" +endif +else + CONFIG_HOME=$(XDG_CONFIG_HOME) +endif hello: @echo "JR,the JSON Random Generator" - CONFIG_HOME=$XDG_CONFIG_HOME - ifeq (CONFIG_HOME, "" ) - HOME = "~" - endif + @echo " Version: $(VERSION)" + @echo " Go Version: $(GOVERSION)" + @echo " Build User: $(USER)" + @echo " Build Time: $(TIME)" + @echo " Detected OS: $(detectedOS)" + @echo " Config Home: $(CONFIG_HOME)" + install-gogen: go install github.com/actgardner/gogen-avro/v10/cmd/...@latest #go install github.com/hamba/avro/v2/cmd/avrogen@latest @@ -18,7 +42,11 @@ generate: compile: @echo "Compiling" - go build -v -ldflags="-X 'github.com/ugol/jr/pkg/cmd.Version=$(VERSION)' -X 'github.com/ugol/jr/pkg/cmd.GoVersion=$(GOVERSION)' -X 'github.com/ugol/jr/pkg/cmd.BuildUser=$(USER)' -X 'github.com/ugol/jr/pkg/cmd.BuildTime=$(TIME)'" -o build/jr jr.go + go build -v -ldflags="-X 'github.com/ugol/jr/pkg/cmd.Version=$(VERSION)' \ + -X 'github.com/ugol/jr/pkg/cmd.GoVersion=$(GOVERSION)' \ + -X 'github.com/ugol/jr/pkg/cmd.BuildUser=$(USER)' \ + -X 'github.com/ugol/jr/pkg/cmd.BuildTime=$(TIME)'" \ + -o build/jr jr.go run: compile ./build/jr @@ -49,13 +77,16 @@ help: hello @echo '' copy_templates: - mkdir -p $CONFIG_HOME/.jr/kafka && cp -r templates $CONFIG_HOME/.jr/ && cp -r pkg/producers/kafka/*.properties.example $CONFIG_HOME/.jr/kafka/ + mkdir -p $(CONFIG_HOME)/$(JR_HOME)/kafka && \ + cp -r templates $(CONFIG_HOME)/$(JR_HOME) && \ + cp -r pkg/producers/kafka/*.properties.example $(CONFIG_HOME)/$(JR_HOME)/kafka/ copy_config: - mkdir -p $CONFIG_HOME/.jr && cp config/* $CONFIG_HOME/.jr/ + mkdir -p $(CONFIG_HOME)/$(JR_HOME) && \ + cp config/* $(CONFIG_HOME)/$(JR_HOME)/ install: install build/jr /usr/local/bin all: hello install-gogen generate compile -all_offline: hello generate compile \ No newline at end of file +all_offline: hello generate compile From b6818785fbbad9ae99cdb1f13857c03e4ea723be Mon Sep 17 00:00:00 2001 From: ugol Date: Sat, 17 Aug 2024 09:09:40 +0200 Subject: [PATCH 41/51] added system dir and user dir --- Makefile | 41 ++++++++++++++++++++++++++++---------- pkg/cmd/root.go | 9 +++++---- pkg/cmd/templateList.go | 2 +- pkg/cmd/templateShow.go | 2 +- pkg/constants/constants.go | 7 ++++--- pkg/emitter/emitter.go | 2 +- pkg/functions/functions.go | 2 +- 7 files changed, 44 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index a13211d2..23bc7575 100644 --- a/Makefile +++ b/Makefile @@ -12,16 +12,36 @@ else endif ifeq ($(detectedOS), Darwin) - CONFIG_HOME="$(HOME)/Library/Application Support" + JR_SYSTEM_DIR="$(HOME)/Library/Application Support" endif ifeq ($(detectedOS), Linux) - CONFIG_HOME="$(HOME)/.config" + JR_SYSTEM_DIR="$(HOME)/.config" endif ifeq ($(detectedOS), Windows_NT) - CONFIG_HOME="$(LOCALAPPDATA)" + JR_SYSTEM_DIR="$(LOCALAPPDATA)" endif else - CONFIG_HOME=$(XDG_CONFIG_HOME) + JR_SYSTEM_DIR=$(XDG_CONFIG_HOME) +endif + +ifndef XDG_DATA_HOME +ifeq ($(OS), Windows_NT) + detectedOS := Windows +else + detectedOS := $(shell sh -c 'uname 2>/dev/null || echo Unknown') +endif + +ifeq ($(detectedOS), Darwin) + JR_USER_DIR="$(HOME)/.local/share" +endif +ifeq ($(detectedOS), Linux) + JR_USER_DIR="$(HOME)/.local/share" +endif +ifeq ($(detectedOS), Windows_NT) + JR_USER_DIR="$(LOCALAPPDATA)" //@TODO +endif +else + JR_USER_DIR=$(XDG_DATA_HOME) endif hello: @@ -31,7 +51,8 @@ hello: @echo " Build User: $(USER)" @echo " Build Time: $(TIME)" @echo " Detected OS: $(detectedOS)" - @echo " Config Home: $(CONFIG_HOME)" + @echo " JR System Dir: $(JR_SYSTEM_DIR)" + @echo " JR User Dir: $(JR_USER_DIR)" install-gogen: go install github.com/actgardner/gogen-avro/v10/cmd/...@latest @@ -77,13 +98,13 @@ help: hello @echo '' copy_templates: - mkdir -p $(CONFIG_HOME)/$(JR_HOME)/kafka && \ - cp -r templates $(CONFIG_HOME)/$(JR_HOME) && \ - cp -r pkg/producers/kafka/*.properties.example $(CONFIG_HOME)/$(JR_HOME)/kafka/ + mkdir -p $(JR_SYSTEM_DIR)/$(JR_HOME)/kafka && \ + cp -r templates $(JR_SYSTEM_DIR)/$(JR_HOME) && \ + cp -r pkg/producers/kafka/*.properties.example $(JR_SYSTEM_DIR)/$(JR_HOME)/kafka/ copy_config: - mkdir -p $(CONFIG_HOME)/$(JR_HOME) && \ - cp config/* $(CONFIG_HOME)/$(JR_HOME)/ + mkdir -p $(JR_SYSTEM_DIR)/$(JR_HOME) && \ + cp config/* $(JR_SYSTEM_DIR)/$(JR_HOME)/ install: install build/jr /usr/local/bin diff --git a/pkg/cmd/root.go b/pkg/cmd/root.go index 5afa922e..3a7addc5 100644 --- a/pkg/cmd/root.go +++ b/pkg/cmd/root.go @@ -58,7 +58,8 @@ func init() { ID: "server", Title: "HTTP Server", }) - rootCmd.PersistentFlags().StringVar(&constants.JRhome, "home", "", "JR home dir") + rootCmd.PersistentFlags().StringVar(&constants.JR_SYSTEM_DIR, "system_dir", "", "JR system dir") + rootCmd.PersistentFlags().StringVar(&constants.JR_USER_DIR, "user_dir", "", "JR user dir") } func initConfig() { @@ -73,10 +74,10 @@ func initConfig() { viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) viper.AutomaticEnv() bindFlags(rootCmd, viper.GetViper()) - if constants.JRhome == "" { - constants.JRhome = constants.DEFAULT_HOMEDIR + if constants.JR_SYSTEM_DIR == "" { + constants.JR_SYSTEM_DIR = constants.SYSTEM_DIR } - viper.AddConfigPath(constants.JRhome) + viper.AddConfigPath(constants.JR_SYSTEM_DIR) if err := viper.ReadInConfig(); err == nil { log.Debug().Str("file", viper.ConfigFileUsed()).Msg("JR configuration") diff --git a/pkg/cmd/templateList.go b/pkg/cmd/templateList.go index f7531c14..86e8058b 100644 --- a/pkg/cmd/templateList.go +++ b/pkg/cmd/templateList.go @@ -42,7 +42,7 @@ var templateListCmd = &cobra.Command{ fmt.Println("List of available JR templates:") fmt.Println() - templateDir := os.ExpandEnv(fmt.Sprintf("%s/%s", constants.JRhome, "templates")) + templateDir := os.ExpandEnv(fmt.Sprintf("%s/%s", constants.JR_SYSTEM_DIR, "templates")) if _, err := os.Stat(templateDir); os.IsNotExist(err) { return diff --git a/pkg/cmd/templateShow.go b/pkg/cmd/templateShow.go index ba2d4c25..a49e206c 100644 --- a/pkg/cmd/templateShow.go +++ b/pkg/cmd/templateShow.go @@ -43,7 +43,7 @@ var templateShowCmd = &cobra.Command{ } noColor, _ := cmd.Flags().GetBool("nocolor") - templateDir := os.ExpandEnv(fmt.Sprintf("%s/%s", constants.JRhome, "templates")) + templateDir := os.ExpandEnv(fmt.Sprintf("%s/%s", constants.JR_SYSTEM_DIR, "templates")) templatePath := fmt.Sprintf("%s/%s.tpl", templateDir, args[0]) templateScript, err := os.ReadFile(templatePath) if err != nil { diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index fddfb28e..f66bcc84 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -27,15 +27,16 @@ import ( "github.com/adrg/xdg" ) -var JRhome string +var JR_SYSTEM_DIR string +var JR_USER_DIR string const NUM = 1 const LOCALE = "us" const FREQUENCY = -1 const INFINITE = 1<<63 - 1 -// var DEFAULT_SYSTEMDIR = xdg.DataHome + "jr" -var DEFAULT_HOMEDIR = fmt.Sprintf("%s%c%s", xdg.ConfigHome, os.PathSeparator, "jr") +var SYSTEM_DIR = fmt.Sprintf("%s%c%s", xdg.ConfigHome, os.PathSeparator, "jr") +var USER_DIR = fmt.Sprintf("%s%c%s", xdg.DataHome, os.PathSeparator, "jr") const DEFAULT_KEY = "null" const DEFAULT_OUTPUT = "stdout" diff --git a/pkg/emitter/emitter.go b/pkg/emitter/emitter.go index da4f377b..2850862f 100644 --- a/pkg/emitter/emitter.go +++ b/pkg/emitter/emitter.go @@ -69,7 +69,7 @@ func (e *Emitter) Initialize(conf configuration.GlobalConfiguration) { templateName := e.ValueTemplate if e.EmbeddedTemplate == "" { - path := os.ExpandEnv(fmt.Sprintf("%s/%s", constants.JRhome, "templates")) + path := os.ExpandEnv(fmt.Sprintf("%s/%s", constants.JR_SYSTEM_DIR, "templates")) templateFullPath := fmt.Sprintf("%s/%s.tpl", path, templateName) vt, err := os.ReadFile(templateFullPath) e.EmbeddedTemplate = string(vt) diff --git a/pkg/functions/functions.go b/pkg/functions/functions.go index ca4f12b4..9186f67c 100644 --- a/pkg/functions/functions.go +++ b/pkg/functions/functions.go @@ -391,7 +391,7 @@ func Maxint(a, b int) int { // Cache is used to internally Cache data from word files func Cache(name string) (bool, error) { - templateDir := fmt.Sprintf("%s/%s", constants.JRhome, "templates") + templateDir := fmt.Sprintf("%s/%s", constants.JR_SYSTEM_DIR, "templates") v := data[name] if v == nil { From 5f6c7e879ba885de414b348013f7a768254cbf77 Mon Sep 17 00:00:00 2001 From: ugol Date: Sat, 17 Aug 2024 09:23:29 +0200 Subject: [PATCH 42/51] fixed some refs to old JR_HOME --- Readme.md | 4 ++-- pkg/cmd/templateList.go | 2 +- pkg/cmd/templateRun.go | 2 +- pkg/cmd/templateShow.go | 2 +- pkg/functions/functionsDescription.go | 12 ++++++------ 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Readme.md b/Readme.md index d863745f..15732a84 100644 --- a/Readme.md +++ b/Readme.md @@ -50,10 +50,10 @@ JR is very straightforward to use. Here are some examples: ```bash jr template list ```` -Templates are in the directory `$JR_HOME/templates`. JR_HOME defaults to `~/.jr` and can be changed to a different dir, for example: +Templates are in the directory `$JR_SYSTEM_DIR/templates`. JR_SYSTEM_DIR defaults to `$XDG_CONFIGDIR` and can be changed to a different dir, for example: ```bash -JR_HOME=~/jrconfig/ jr template list +JR_SYSTEM_DIR=~/jrconfig/ jr template list ```` Templates with parsing issues are showed in red, Templates with no parsing issues are showed in green diff --git a/pkg/cmd/templateList.go b/pkg/cmd/templateList.go index 86e8058b..1ff321c0 100644 --- a/pkg/cmd/templateList.go +++ b/pkg/cmd/templateList.go @@ -35,7 +35,7 @@ import ( var templateListCmd = &cobra.Command{ Use: "list", Short: "List all available templates", - Long: `List all available templates, which are in '$JR_HOME/templates' directory`, + Long: `List all available templates, which are in '$JR_SYSTEM_DIR/templates' directory`, Run: func(cmd *cobra.Command, args []string) { fmt.Println() diff --git a/pkg/cmd/templateRun.go b/pkg/cmd/templateRun.go index 60e83c53..4560d48c 100644 --- a/pkg/cmd/templateRun.go +++ b/pkg/cmd/templateRun.go @@ -36,7 +36,7 @@ var templateRunCmd = &cobra.Command{ Use: "run [template]", Short: "Execute a template", Long: `Execute a template. - Without any other flag, [template] is just the name of a template in the templates directory, which is '$JR_HOME/templates'. Example: + Without any other flag, [template] is just the name of a template in the templates directory, which is '$JR_SYSTEM_DIR/templates'. Example: jr template run net_device With the --embedded flag, [template] is a string containing a full template. Example: jr template run --template "{{name}}" diff --git a/pkg/cmd/templateShow.go b/pkg/cmd/templateShow.go index a49e206c..bf9c7bc3 100644 --- a/pkg/cmd/templateShow.go +++ b/pkg/cmd/templateShow.go @@ -34,7 +34,7 @@ import ( var templateShowCmd = &cobra.Command{ Use: "show [template]", Short: "Show a template", - Long: `Show a template. Templates must be in templates directory, which is '$JR_HOME/templates'`, + Long: `Show a template. Templates must be in templates directory, which is '$JR_SYSTEM_DIR/templates'`, Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { if len(args) == 0 { diff --git a/pkg/functions/functionsDescription.go b/pkg/functions/functionsDescription.go index 52c422aa..24e25860 100644 --- a/pkg/functions/functionsDescription.go +++ b/pkg/functions/functionsDescription.go @@ -445,7 +445,7 @@ var funcDesc = map[string]FunctionDescription{ "from": { Name: "from", Category: "text utilities", - Description: "returns a random string from a list of strings in a file. Files are in '$JR_HOME/templates/data/locale'", + Description: "returns a random string from a list of strings in a file. Files are in '$JR_SYSTEM_DIR/templates/data/locale'", Parameters: "set string", Localizable: true, Return: "string", @@ -455,7 +455,7 @@ var funcDesc = map[string]FunctionDescription{ "from_at": { Name: "from_at", Category: "text utilities", - Description: "returns a string at a given position in a list of strings in a file. Files are in '$JR_HOME/templates/data/locale'", + Description: "returns a string at a given position in a list of strings in a file. Files are in '$JR_SYSTEM_DIR/templates/data/locale'", Parameters: "index int", Localizable: true, Return: "string", @@ -465,7 +465,7 @@ var funcDesc = map[string]FunctionDescription{ "from_n": { Name: "from_n", Category: "text utilities", - Description: "return a subset of elements in a list of string in a file. Files are in '$JR_HOME/templates/data/locale'", + Description: "return a subset of elements in a list of string in a file. Files are in '$JR_SYSTEM_DIR/templates/data/locale'", Parameters: "set string, number int", Localizable: true, Return: "[]string", @@ -475,7 +475,7 @@ var funcDesc = map[string]FunctionDescription{ "from_shuffle": { Name: "from_shuffle", Category: "text utilities", - Description: "returns a shuffled list of strings in a file. Files are in '$JR_HOME/templates/data/locale'", + Description: "returns a shuffled list of strings in a file. Files are in '$JR_SYSTEM_DIR/templates/data/locale'", Parameters: "set string", Localizable: true, Return: "[]string", @@ -679,7 +679,7 @@ var funcDesc = map[string]FunctionDescription{ Category: "text utilities", Parameters: "set string", Localizable: true, - Description: "returns the length a list of strings in a file. Files are in '$JR_HOME/templates/data/locale'", + Description: "returns the length a list of strings in a file. Files are in '$JR_SYSTEM_DIR/templates/data/locale'", Return: "string", Example: "jr template run --embedded '{{len \"city\"}}'", Output: "46", @@ -909,7 +909,7 @@ var funcDesc = map[string]FunctionDescription{ Category: "text utilities", Parameters: "set string", Localizable: true, - Description: "returns a random index from a list of strings in a file. Files are in '$JR_HOME/templates/data/locale'", + Description: "returns a random index from a list of strings in a file. Files are in '$JR_SYSTEM_DIR/templates/data/locale'", Return: "string", Example: "jr template run --embedded '{{random_index \"city\"}}'", Output: "12", From 4e5bf09b8b1746a758f77284aec2df068f75cb53 Mon Sep 17 00:00:00 2001 From: ugol Date: Sat, 17 Aug 2024 09:25:56 +0200 Subject: [PATCH 43/51] fixed some refs to net-device instead of net_device in godoc --- jr.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/jr.go b/jr.go index 15afae56..7f91db06 100644 --- a/jr.go +++ b/jr.go @@ -26,24 +26,24 @@ JR is a CLI program that helps you to create quality random data for your applic JR is very straightforward to use. To list existing templates: > jr list -Templates are in the directory $HOME/.jr/templates. -You can override with the --templatePath command flag Templates with parsing issues are showed in red, Templates with no parsing issues are showed in green +Templates are in the directory $JR_SYSTEM_DIR/.jr/templates. +You can override with the --system_dir command flag. Templates with parsing issues are showed in red, Templates with no parsing issues are showed in green -To use for example one of the predefined templates, net-device: +To use for example one of the predefined templates, net_device: -> jr template run net-device +> jr template run net_device -Using -n option you can create more data in each pass. This example creates 3 net-device objects at once: +Using -n option you can create more data in each pass. This example creates 3 net_device objects at once: -> jr template run net-device -n 3 +> jr template run net_device -n 3 -Using --frequency option you can repeat the creation every f milliseconds. This example creates 2 net-device every second, for ever: +Using --frequency option you can repeat the creation every f milliseconds. This example creates 2 net_device every second, for ever: -> jr template run net-device -n 2 -f 1s +> jr template run net_device -n 2 -f 1s -Using --duration option you can time bound the entire object creation. This example creates 2 net-device every 100ms for 1 minute: +Using --duration option you can time bound the entire object creation. This example creates 2 net_device every 100ms for 1 minute: -> jr template run net-device -n 2 -f 100ms -d 1m +> jr template run net_device -n 2 -f 100ms -d 1m */ package main From 5280d3579870bc84793d896894f2341c88a29dbf Mon Sep 17 00:00:00 2001 From: ugol Date: Sat, 17 Aug 2024 09:26:30 +0200 Subject: [PATCH 44/51] fixed some refs to net-device instead of net_device in godoc --- jr.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jr.go b/jr.go index 7f91db06..3dda03b0 100644 --- a/jr.go +++ b/jr.go @@ -26,7 +26,7 @@ JR is a CLI program that helps you to create quality random data for your applic JR is very straightforward to use. To list existing templates: > jr list -Templates are in the directory $JR_SYSTEM_DIR/.jr/templates. +Templates are in the directory $JR_SYSTEM_DIR/jr/templates. You can override with the --system_dir command flag. Templates with parsing issues are showed in red, Templates with no parsing issues are showed in green To use for example one of the predefined templates, net_device: From 03f41da5941900630eb81e2addcfc734091eda9f Mon Sep 17 00:00:00 2001 From: Vincenzo Date: Sat, 17 Aug 2024 09:48:31 +0200 Subject: [PATCH 45/51] feat: add log_level (defaults to panic) --- pkg/cmd/root.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pkg/cmd/root.go b/pkg/cmd/root.go index 3a7addc5..3abed7e1 100644 --- a/pkg/cmd/root.go +++ b/pkg/cmd/root.go @@ -26,6 +26,7 @@ import ( "strings" "time" + "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -35,6 +36,8 @@ import ( "github.com/ugol/jr/pkg/functions" ) +var logLevel = "panic" + var rootCmd = &cobra.Command{ Use: "jr", Short: "jr, the data random generator", @@ -60,10 +63,18 @@ func init() { }) rootCmd.PersistentFlags().StringVar(&constants.JR_SYSTEM_DIR, "system_dir", "", "JR system dir") rootCmd.PersistentFlags().StringVar(&constants.JR_USER_DIR, "user_dir", "", "JR user dir") + rootCmd.PersistentFlags().StringVar(&logLevel, "log_level", "panic", "HR Log Level") } func initConfig() { + // setting zerolog level + zlogLevel, err := zerolog.ParseLevel(logLevel) + if err != nil { + zlogLevel = zerolog.PanicLevel + } + zerolog.SetGlobalLevel(zlogLevel) + viper.SetConfigName("jrconfig") viper.SetConfigType("json") viper.AddConfigPath(".") @@ -84,7 +95,7 @@ func initConfig() { } else { log.Error().Err(err).Msg("JR configuration not found") } - err := viper.UnmarshalKey("global", &configuration.GlobalCfg) + err = viper.UnmarshalKey("global", &configuration.GlobalCfg) if err != nil { log.Error().Err(err).Msg("Failed to unmarshal global configuration") } From 33dc1baaf80dd304321025e48863e983ea81433a Mon Sep 17 00:00:00 2001 From: Vincenzo Date: Sat, 17 Aug 2024 09:55:54 +0200 Subject: [PATCH 46/51] feat: change log level to fatal --- pkg/cmd/root.go | 4 ++-- pkg/constants/constants.go | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/cmd/root.go b/pkg/cmd/root.go index 3abed7e1..c616078d 100644 --- a/pkg/cmd/root.go +++ b/pkg/cmd/root.go @@ -36,7 +36,7 @@ import ( "github.com/ugol/jr/pkg/functions" ) -var logLevel = "panic" +var logLevel = constants.DEFAULT_LOG_LEVEL var rootCmd = &cobra.Command{ Use: "jr", @@ -63,7 +63,7 @@ func init() { }) rootCmd.PersistentFlags().StringVar(&constants.JR_SYSTEM_DIR, "system_dir", "", "JR system dir") rootCmd.PersistentFlags().StringVar(&constants.JR_USER_DIR, "user_dir", "", "JR user dir") - rootCmd.PersistentFlags().StringVar(&logLevel, "log_level", "panic", "HR Log Level") + rootCmd.PersistentFlags().StringVar(&logLevel, "log_level", constants.DEFAULT_LOG_LEVEL, "HR Log Level") } func initConfig() { diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index f66bcc84..81624172 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -51,3 +51,5 @@ const DEFAULT_EMITTER_NAME = "cli" const DEFAULT_VALUE_TEMPLATE = "user" const DEFAULT_TOPIC = "test" const DEFAULT_HTTP_PORT = 7482 + +const DEFAULT_LOG_LEVEL = "fatal" From 3daf9b37d559bc4b715e9123d1b0fe48c97cd10b Mon Sep 17 00:00:00 2001 From: ugol Date: Sat, 17 Aug 2024 10:37:59 +0200 Subject: [PATCH 47/51] fixed XDG_DATA_DIRS --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 23bc7575..77bc59a9 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ USER=$(shell id -u -n) TIME=$(shell date) JR_HOME=jr -ifndef XDG_CONFIG_HOME +ifndef XDG_DATA_DIRS ifeq ($(OS), Windows_NT) detectedOS := Windows else @@ -21,7 +21,7 @@ ifeq ($(detectedOS), Windows_NT) JR_SYSTEM_DIR="$(LOCALAPPDATA)" endif else - JR_SYSTEM_DIR=$(XDG_CONFIG_HOME) + JR_SYSTEM_DIR=$(XDG_DATA_DIRS) endif ifndef XDG_DATA_HOME From 236b5955938f80af68fa307916d9dfb71f388668 Mon Sep 17 00:00:00 2001 From: ugol Date: Sat, 17 Aug 2024 13:25:25 +0200 Subject: [PATCH 48/51] bug fixed in jr function list --- pkg/cmd/functionList.go | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/pkg/cmd/functionList.go b/pkg/cmd/functionList.go index eb1e8d08..7fdde2b9 100644 --- a/pkg/cmd/functionList.go +++ b/pkg/cmd/functionList.go @@ -39,18 +39,6 @@ var functionListCmd = &cobra.Command{ }, } -/* -var functionManMarkdownCmd = &cobra.Command{ - Use: "manmd", - Short: "describes available functions as markdown", - Long: "describes available functions as markdown. Example usage:\n" + - "jr function manmd", - Run: func(cmd *cobra.Command, args []string) { - doList(cmd, args, false) - }, -} -*/ - var functionManCmd = &cobra.Command{ Use: "man", Short: "describes available functions", @@ -68,13 +56,13 @@ func doList(cmd *cobra.Command, args []string) { isMarkdown, _ := cmd.Flags().GetBool("markdown") noColor, _ := cmd.Flags().GetBool("nocolor") - if category { + if category && len(args) > 0 { for k, v := range functions.DescriptionMap() { if strings.Contains(v.Category, args[0]) { printFunction(k, isMarkdown, noColor) } } - } else if find { + } else if find && len(args) > 0 { for k, v := range functions.DescriptionMap() { if strings.Contains(v.Description, args[0]) || strings.Contains(v.Name, args[0]) { printFunction(k, isMarkdown, noColor) From c6e006671d9bb36973756cd27e24e7b406073906 Mon Sep 17 00:00:00 2001 From: ugol Date: Sun, 18 Aug 2024 01:53:03 +0200 Subject: [PATCH 49/51] added sorted output, total number of functions in category and 4 missing function descriptions --- pkg/cmd/functionList.go | 63 +++++++++++++++++---------- pkg/functions/functionsDescription.go | 42 +++++++++++++++++- 2 files changed, 81 insertions(+), 24 deletions(-) diff --git a/pkg/cmd/functionList.go b/pkg/cmd/functionList.go index 7fdde2b9..8991fa96 100644 --- a/pkg/cmd/functionList.go +++ b/pkg/cmd/functionList.go @@ -26,6 +26,7 @@ import ( "github.com/ugol/jr/pkg/functions" "os" "os/exec" + "slices" "strings" ) @@ -57,17 +58,21 @@ func doList(cmd *cobra.Command, args []string) { noColor, _ := cmd.Flags().GetBool("nocolor") if category && len(args) > 0 { + var functionNames []string for k, v := range functions.DescriptionMap() { if strings.Contains(v.Category, args[0]) { - printFunction(k, isMarkdown, noColor) + functionNames = append(functionNames, k) } } + sortAndPrint(functionNames, isMarkdown, noColor) } else if find && len(args) > 0 { + var functionNames []string for k, v := range functions.DescriptionMap() { if strings.Contains(v.Description, args[0]) || strings.Contains(v.Name, args[0]) { - printFunction(k, isMarkdown, noColor) + functionNames = append(functionNames, k) } } + sortAndPrint(functionNames, isMarkdown, noColor) } else if len(args) == 1 { if run { @@ -83,15 +88,30 @@ func doList(cmd *cobra.Command, args []string) { printFunction(args[0], isMarkdown, noColor) } } else { - count := 0 - for k := range functions.FunctionsMap() { - count++ - printFunction(k, isMarkdown, noColor) + + //l := len(functions.FunctionsMap()) + l := len(functions.DescriptionMap()) + functionNames := make([]string, l) + + i := 0 + for k := range functions.DescriptionMap() { + functionNames[i] = k + i++ } - fmt.Println() - fmt.Printf("Total functions: %d\n", count) + sortAndPrint(functionNames, isMarkdown, noColor) + + } + fmt.Println() +} + +func sortAndPrint(functionNames []string, isMarkdown bool, noColor bool) { + slices.Sort(functionNames) + for _, k := range functionNames { + printFunction(k, isMarkdown, noColor) + //fmt.Println(k) } fmt.Println() + fmt.Printf("Total functions: %d\n", len(functionNames)) } func printFunction(name string, isMarkdown bool, noColor bool) (functions.FunctionDescription, bool) { @@ -104,21 +124,8 @@ func printFunction(name string, isMarkdown bool, noColor bool) (functions.Functi Reset = "\033[0m" } - if !isMarkdown { - - if found { - fmt.Println() - fmt.Printf("%sName: %s%s\n", Cyan, Reset, f.Name) - fmt.Printf("%sCategory: %s%s\n", Cyan, Reset, f.Category) - fmt.Printf("%sDescription: %s%s\n", Cyan, Reset, f.Description) - fmt.Printf("%sParameters: %s%s\n", Cyan, Reset, f.Parameters) - fmt.Printf("%sLocalizable: %s%v\n", Cyan, Reset, f.Localizable) - fmt.Printf("%sReturn: %s%s\n", Cyan, Reset, f.Return) - fmt.Printf("%sExample: %s%s\n", Cyan, Reset, f.Example) - fmt.Printf("%sOutput: %s%s\n", Cyan, Reset, f.Output) - } - } else { - if found { + if found { + if isMarkdown { fmt.Println() fmt.Printf("## Name: %s \n", f.Name) fmt.Printf("**Category:** %s\\\n", f.Category) @@ -133,6 +140,16 @@ func printFunction(name string, isMarkdown bool, noColor bool) (functions.Functi fmt.Printf("**Return:** `%s`\\\n", f.Return) fmt.Printf("**Example:** `%s`\\\n", f.Example) fmt.Printf("**Output:** `%s`\n", f.Output) + } else { + fmt.Println() + fmt.Printf("%sName: %s%s\n", Cyan, Reset, f.Name) + fmt.Printf("%sCategory: %s%s\n", Cyan, Reset, f.Category) + fmt.Printf("%sDescription: %s%s\n", Cyan, Reset, f.Description) + fmt.Printf("%sParameters: %s%s\n", Cyan, Reset, f.Parameters) + fmt.Printf("%sLocalizable: %s%v\n", Cyan, Reset, f.Localizable) + fmt.Printf("%sReturn: %s%s\n", Cyan, Reset, f.Return) + fmt.Printf("%sExample: %s%s\n", Cyan, Reset, f.Example) + fmt.Printf("%sOutput: %s%s\n", Cyan, Reset, f.Output) } } return f, found diff --git a/pkg/functions/functionsDescription.go b/pkg/functions/functionsDescription.go index 6201c00c..317d93da 100644 --- a/pkg/functions/functionsDescription.go +++ b/pkg/functions/functionsDescription.go @@ -512,6 +512,16 @@ var funcDesc = map[string]FunctionDescription{ Example: "jr template run --embedded '{{set_v \"id\" \"12770\"}}{{get_v \"id\"}}'", Output: "12770", }, + "get_v_from_list_at_index": { + Name: "get_v_from_list_at_index", + Category: "context", + Description: "returns a specific value from a list. The list must be set with 'add_v_to_list', usually in an other template", + Parameters: "name string index int", + Localizable: false, + Return: "string", + Example: "jr template run --embedded '{{add_v_to_list \"ids\" \"12770\"}}{{get_v_from_list_at_index \"ids\" 0}}'", + Output: "12770", + }, "http_method": { Name: "http_method", Category: "network", @@ -728,6 +738,16 @@ var funcDesc = map[string]FunctionDescription{ Name: "max", Category: "math", Description: "returns the maximum of two numbers", + Parameters: "first int|float, second int|float", + Localizable: false, + Return: "int", + Example: "jr template run --embedded '{{max 10.3 2.4}}'", + Output: "10.3", + }, + "maxint": { + Name: "max", + Category: "math", + Description: "returns the maximum of two ints", Parameters: "first int, second int", Localizable: false, Return: "int", @@ -748,7 +768,17 @@ var funcDesc = map[string]FunctionDescription{ Name: "min", Category: "math", Description: "returns the minimum of two numbers", - Parameters: "min int, max int", + Parameters: "first int|float, second int|float", + Localizable: false, + Return: "int", + Example: "jr template run --embedded '{{min 10.1 2.3}}'", + Output: "2.3", + }, + "minint": { + Name: "min", + Category: "math", + Description: "returns the minimum of two ints", + Parameters: "first int, second int", Localizable: false, Return: "int", Example: "jr template run --embedded '{{min 10 2}}'", @@ -994,6 +1024,16 @@ var funcDesc = map[string]FunctionDescription{ Example: "jr template run --embedded '{{replaceall \"hello world\" \"hello\" \"goodbye\"}}'", Output: "goodbye world", }, + "seed": { + Name: "seed", + Category: "utilities", + Description: "set seed directly in a template", + Parameters: "rndSeed int64", + Localizable: false, + Return: "", + Example: "jr template run --embedded '{{seed 12345}}'", + Output: "", + }, "sedol": { Name: "sedol", Category: "finance", From 61b383f20f24c800a756cbe05be8eb5c8d86c821 Mon Sep 17 00:00:00 2001 From: ugol Date: Sun, 18 Aug 2024 02:30:32 +0200 Subject: [PATCH 50/51] switched category to text and -c now must be a perfect match to show --- pkg/cmd/functionList.go | 2 +- pkg/functions/functionsDescription.go | 66 +++++++++++++-------------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/pkg/cmd/functionList.go b/pkg/cmd/functionList.go index 8991fa96..2292dd6e 100644 --- a/pkg/cmd/functionList.go +++ b/pkg/cmd/functionList.go @@ -60,7 +60,7 @@ func doList(cmd *cobra.Command, args []string) { if category && len(args) > 0 { var functionNames []string for k, v := range functions.DescriptionMap() { - if strings.Contains(v.Category, args[0]) { + if v.Category == args[0] { functionNames = append(functionNames, k) } } diff --git a/pkg/functions/functionsDescription.go b/pkg/functions/functionsDescription.go index 317d93da..b5846e58 100644 --- a/pkg/functions/functionsDescription.go +++ b/pkg/functions/functionsDescription.go @@ -94,7 +94,7 @@ var funcDesc = map[string]FunctionDescription{ }, "atoi": { Name: "atoi", - Category: "text utilities", + Category: "text", Description: "converts a string to an integer", Parameters: "string", Localizable: false, @@ -104,7 +104,7 @@ var funcDesc = map[string]FunctionDescription{ }, "itoa": { Name: "itoa", - Category: "text utilities", + Category: "text", Description: "converts an integer to a string", Parameters: "int", Localizable: false, @@ -114,7 +114,7 @@ var funcDesc = map[string]FunctionDescription{ }, "concat": { Name: "concat", - Category: "text utilities", + Category: "text", Description: "converts an integer to a string", Parameters: "string string", Localizable: false, @@ -404,7 +404,7 @@ var funcDesc = map[string]FunctionDescription{ }, "first": { Name: "first", - Category: "text utilities", + Category: "text", Description: "returns the first character of a string", Parameters: "text string", Localizable: false, @@ -414,7 +414,7 @@ var funcDesc = map[string]FunctionDescription{ }, "firstword": { Name: "firstword", - Category: "text utilities", + Category: "text", Description: "returns the first word of a text", Parameters: "text string", Localizable: false, @@ -444,7 +444,7 @@ var funcDesc = map[string]FunctionDescription{ }, "from": { Name: "from", - Category: "text utilities", + Category: "text", Description: "returns a random string from a list of strings in a file. Files are in '$JR_SYSTEM_DIR/templates/data/locale'", Parameters: "set string", Localizable: true, @@ -454,7 +454,7 @@ var funcDesc = map[string]FunctionDescription{ }, "from_at": { Name: "from_at", - Category: "text utilities", + Category: "text", Description: "returns a string at a given position in a list of strings in a file. Files are in '$JR_SYSTEM_DIR/templates/data/locale'", Parameters: "index int", Localizable: true, @@ -464,7 +464,7 @@ var funcDesc = map[string]FunctionDescription{ }, "from_n": { Name: "from_n", - Category: "text utilities", + Category: "text", Description: "return a subset of elements in a list of string in a file. Files are in '$JR_SYSTEM_DIR/templates/data/locale'", Parameters: "set string, number int", Localizable: true, @@ -474,7 +474,7 @@ var funcDesc = map[string]FunctionDescription{ }, "from_shuffle": { Name: "from_shuffle", - Category: "text utilities", + Category: "text", Description: "returns a shuffled list of strings in a file. Files are in '$JR_SYSTEM_DIR/templates/data/locale'", Parameters: "set string", Localizable: true, @@ -646,7 +646,7 @@ var funcDesc = map[string]FunctionDescription{ }, "join": { Name: "join", - Category: "text utilities", + Category: "text", Description: "joins a list of strings with a separator", Parameters: "strings []string", Localizable: false, @@ -666,7 +666,7 @@ var funcDesc = map[string]FunctionDescription{ }, "index_of": { Name: "index_of", - Category: "text utilities", + Category: "text", Description: "returns the index of s in 'name' file ", Parameters: "s string, name string", Localizable: false, @@ -686,7 +686,7 @@ var funcDesc = map[string]FunctionDescription{ }, "len": { Name: "len", - Category: "text utilities", + Category: "text", Parameters: "set string", Localizable: true, Description: "returns the length a list of strings in a file. Files are in '$JR_SYSTEM_DIR/templates/data/locale'", @@ -706,7 +706,7 @@ var funcDesc = map[string]FunctionDescription{ }, "lower": { Name: "lower", - Category: "text utilities", + Category: "text", Description: "converts a string to lowercase", Parameters: "text string", Localizable: false, @@ -716,7 +716,7 @@ var funcDesc = map[string]FunctionDescription{ }, "lorem": { Name: "lorem", - Category: "text utilities", + Category: "text", Description: "generates a Lorem ipsum string", Parameters: "words int", Localizable: false, @@ -756,7 +756,7 @@ var funcDesc = map[string]FunctionDescription{ }, "markov": { Name: "markov", - Category: "text utilities", + Category: "text", Description: "generates a markov chain from a string", Parameters: "chain int, text string", Localizable: false, @@ -916,7 +916,7 @@ var funcDesc = map[string]FunctionDescription{ }, "random": { Name: "random", - Category: "text utilities", + Category: "text", Description: "returns a random string from a list of strings", Parameters: "list []string", Localizable: false, @@ -926,7 +926,7 @@ var funcDesc = map[string]FunctionDescription{ }, "randoms": { Name: "randoms", - Category: "text utilities", + Category: "text", Parameters: "list string", Localizable: false, Description: "returns a random strings from a | separated list string", @@ -936,7 +936,7 @@ var funcDesc = map[string]FunctionDescription{ }, "random_index": { Name: "random_index", - Category: "text utilities", + Category: "text", Parameters: "set string", Localizable: true, Description: "returns a random index from a list of strings in a file. Files are in '$JR_SYSTEM_DIR/templates/data/locale'", @@ -946,7 +946,7 @@ var funcDesc = map[string]FunctionDescription{ }, "random_string": { Name: "random_string", - Category: "text utilities", + Category: "text", Description: "returns a random string long between min and max characters", Parameters: "min int, max int, vocabulary string", Localizable: false, @@ -956,7 +956,7 @@ var funcDesc = map[string]FunctionDescription{ }, "random_string_vocabulary": { Name: "random_string_vocabulary", - Category: "text utilities", + Category: "text", Description: "returns a random string long between min and max characters using a vocabulary", Parameters: "min int, max int, vocabulary string", Localizable: false, @@ -996,7 +996,7 @@ var funcDesc = map[string]FunctionDescription{ }, "regex": { Name: "regex", - Category: "text utilities", + Category: "text", Description: "returns a random string matching the Regex", Parameters: "regex string", Localizable: false, @@ -1006,7 +1006,7 @@ var funcDesc = map[string]FunctionDescription{ }, "repeat": { Name: "repeat", - Category: "text utilities", + Category: "text", Description: "repeats a string a number of times", Parameters: "text string, number int", Localizable: false, @@ -1016,7 +1016,7 @@ var funcDesc = map[string]FunctionDescription{ }, "replaceall": { Name: "replaceall", - Category: "text utilities", + Category: "text", Description: "replaces all instances of a string with another string", Parameters: "set string, original string, replaced string", Localizable: false, @@ -1046,7 +1046,7 @@ var funcDesc = map[string]FunctionDescription{ }, "sentence": { Name: "sentence", - Category: "text utilities", + Category: "text", Description: "generates a random Sentence of n words", Parameters: "words int", Localizable: false, @@ -1056,7 +1056,7 @@ var funcDesc = map[string]FunctionDescription{ }, "sentence_prefix": { Name: "sentence_prefix", - Category: "text utilities", + Category: "text", Description: "generates a random Sentence of n words, with a prefix length", Parameters: "prefix int, length int", Localizable: false, @@ -1086,7 +1086,7 @@ var funcDesc = map[string]FunctionDescription{ }, "split": { Name: "split", - Category: "text utilities", + Category: "text", Description: "splits a string into a list of strings", Parameters: "text string, separator string", Localizable: false, @@ -1106,7 +1106,7 @@ var funcDesc = map[string]FunctionDescription{ }, "squeezechars": { Name: "squeezechars", - Category: "text utilities", + Category: "text", Description: "removes all instances of a character from a string", Parameters: "set string, chars string", Localizable: false, @@ -1206,7 +1206,7 @@ var funcDesc = map[string]FunctionDescription{ }, "squeeze": { Name: "squeeze", - Category: "text utilities", + Category: "text", Description: "removes all spaces from a string", Parameters: "text string", Localizable: false, @@ -1216,7 +1216,7 @@ var funcDesc = map[string]FunctionDescription{ }, "substr": { Name: "substr", - Category: "text utilities", + Category: "text", Description: "returns a substring of a string", Parameters: "from int, to int, text string", Localizable: false, @@ -1226,7 +1226,7 @@ var funcDesc = map[string]FunctionDescription{ }, "title": { Name: "title", - Category: "text utilities", + Category: "text", Description: "converts a string to title case", Parameters: "text string", Localizable: false, @@ -1236,7 +1236,7 @@ var funcDesc = map[string]FunctionDescription{ }, "trim": { Name: "trim", - Category: "text utilities", + Category: "text", Description: "trims whitespace from a string", Parameters: "text string", Localizable: false, @@ -1246,7 +1246,7 @@ var funcDesc = map[string]FunctionDescription{ }, "trimchars": { Name: "trimchars", - Category: "text utilities", + Category: "text", Description: "trims all characters in the given set from the beginning and end of a string", Parameters: "set string, chars string", Localizable: false, @@ -1266,7 +1266,7 @@ var funcDesc = map[string]FunctionDescription{ }, "upper": { Name: "upper", - Category: "text utilities", + Category: "text", Description: "converts a string to uppercase", Parameters: "text string", Localizable: false, From 37970bc4bcf4968c9382b6a56c8536af9524d427 Mon Sep 17 00:00:00 2001 From: ugol Date: Sun, 18 Aug 2024 02:57:07 +0200 Subject: [PATCH 51/51] minor fixes at md gen --- pkg/cmd/functionList.go | 2 +- pkg/functions/functionsDescription.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cmd/functionList.go b/pkg/cmd/functionList.go index 2292dd6e..0d4bf8e7 100644 --- a/pkg/cmd/functionList.go +++ b/pkg/cmd/functionList.go @@ -127,7 +127,7 @@ func printFunction(name string, isMarkdown bool, noColor bool) (functions.Functi if found { if isMarkdown { fmt.Println() - fmt.Printf("## Name: %s \n", f.Name) + fmt.Printf("### %s \n", f.Name) fmt.Printf("**Category:** %s\\\n", f.Category) fmt.Printf("**Description:** %s\\\n", f.Description) diff --git a/pkg/functions/functionsDescription.go b/pkg/functions/functionsDescription.go index b5846e58..dccbab4f 100644 --- a/pkg/functions/functionsDescription.go +++ b/pkg/functions/functionsDescription.go @@ -1030,7 +1030,7 @@ var funcDesc = map[string]FunctionDescription{ Description: "set seed directly in a template", Parameters: "rndSeed int64", Localizable: false, - Return: "", + Return: " ", Example: "jr template run --embedded '{{seed 12345}}'", Output: "", },