From 47630a925448a1b4cf1345c604fe6a48ae945240 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Thu, 3 Oct 2024 16:58:32 +1000 Subject: [PATCH] feat: Batch support (#779) Co-authored-by: Ryan Cartwright Co-authored-by: David Moore --- cmd/debug.go | 20 +- cmd/run.go | 17 +- cmd/stack.go | 22 +- cmd/start.go | 9 + go.mod | 91 ++-- go.sum | 208 +++++----- pkg/cloud/batch/batch.go | 198 +++++++++ pkg/cloud/cloud.go | 78 +++- pkg/cloud/gateway/gateway.go | 44 ++ pkg/cloud/resources/resources.go | 7 + pkg/cloud/resources/services.go | 21 +- pkg/cloud/schedules/schedules.go | 1 + pkg/collector/batch.go | 278 +++++++++++++ pkg/collector/service.go | 20 + pkg/collector/spec.go | 288 ++++++++++--- pkg/dashboard/dashboard.go | 101 ++++- .../architecture/nodes/BatchNode.tsx | 39 ++ .../components/architecture/nodes/JobNode.tsx | 26 ++ .../src/components/architecture/styles.css | 14 + .../src/components/events/EventsExplorer.tsx | 40 +- .../src/components/events/EventsHistory.tsx | 9 +- .../src/components/events/EventsTreeView.tsx | 16 +- .../components/layout/AppLayout/AppLayout.tsx | 6 + .../frontend/src/lib/hooks/use-history.ts | 9 +- .../lib/utils/generate-architecture-data.ts | 69 ++++ pkg/dashboard/frontend/src/pages/jobs.astro | 8 + pkg/dashboard/frontend/src/types.ts | 20 +- pkg/dashboard/handlers.go | 16 + pkg/dashboard/history.go | 20 +- pkg/preview/feature.go | 1 + pkg/project/batch.go | 388 ++++++++++++++++++ pkg/project/config.go | 38 +- pkg/project/migrations.go | 2 +- pkg/project/project.go | 316 ++++++++++++-- pkg/project/stack/aws.config.yaml | 2 +- pkg/project/stack/awstf.config.yaml | 2 +- pkg/project/stack/azure.config.yaml | 2 +- pkg/project/stack/gcp.config.yaml | 2 +- pkg/project/stack/gcptf.config.yaml | 2 +- 39 files changed, 2139 insertions(+), 311 deletions(-) create mode 100644 pkg/cloud/batch/batch.go create mode 100644 pkg/collector/batch.go create mode 100644 pkg/dashboard/frontend/src/components/architecture/nodes/BatchNode.tsx create mode 100644 pkg/dashboard/frontend/src/components/architecture/nodes/JobNode.tsx create mode 100644 pkg/dashboard/frontend/src/pages/jobs.astro create mode 100644 pkg/project/batch.go diff --git a/cmd/debug.go b/cmd/debug.go index f81a3f7f4..2c29979df 100644 --- a/cmd/debug.go +++ b/cmd/debug.go @@ -21,6 +21,7 @@ import ( "os" "strings" + "github.com/samber/lo" "github.com/spf13/afero" "github.com/spf13/cobra" "google.golang.org/protobuf/encoding/protojson" @@ -64,6 +65,11 @@ var specCmd = &cobra.Command{ buildUpdates, err := proj.BuildServices(fs) tui.CheckErr(err) + batchBuildUpdates, err := proj.BuildBatches(fs) + tui.CheckErr(err) + + allBuildUpdates := lo.FanIn(10, buildUpdates, batchBuildUpdates) + if isNonInteractive() { fmt.Println("building project services") for _, service := range proj.GetServices() { @@ -77,7 +83,7 @@ var specCmd = &cobra.Command{ } } } else { - prog := teax.NewProgram(build.NewModel(buildUpdates, "Building Services")) + prog := teax.NewProgram(build.NewModel(allBuildUpdates, "Building Services")) // blocks but quits once the above updates channel is closed by the build process buildModel, err := prog.Run() tui.CheckErr(err) @@ -91,6 +97,9 @@ var specCmd = &cobra.Command{ serviceRequirements, err := proj.CollectServicesRequirements() tui.CheckErr(err) + batchRequirements, err := proj.CollectBatchRequirements() + tui.CheckErr(err) + additionalEnvFiles := []string{} if debugEnvFile != "" { @@ -106,12 +115,7 @@ var specCmd = &cobra.Command{ envVariables = map[string]string{} } - defaultImageName, ok := proj.DefaultMigrationImage(fs) - if !ok { - defaultImageName = "" - } - - migrationImageContexts, err := collector.GetMigrationImageBuildContexts(serviceRequirements, fs) + migrationImageContexts, err := collector.GetMigrationImageBuildContexts(serviceRequirements, batchRequirements, fs) tui.CheckErr(err) // Build images from contexts and provide updates on the builds @@ -143,7 +147,7 @@ var specCmd = &cobra.Command{ outputFile = "./nitric-spec.json" } - spec, err := collector.ServiceRequirementsToSpec(proj.Name, envVariables, serviceRequirements, defaultImageName) + spec, err := collector.ServiceRequirementsToSpec(proj.Name, envVariables, serviceRequirements, batchRequirements) tui.CheckErr(err) marshaler := protojson.MarshalOptions{ diff --git a/cmd/run.go b/cmd/run.go index 0b8763317..83b6e7fd9 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -119,7 +119,12 @@ var runCmd = &cobra.Command{ updates, err := proj.BuildServices(fs) tui.CheckErr(err) - prog := teax.NewProgram(build.NewModel(updates, "Building Services")) + batchBuildUpdates, err := proj.BuildBatches(fs) + tui.CheckErr(err) + + allBuildUpdates := lo.FanIn(10, updates, batchBuildUpdates) + + prog := teax.NewProgram(build.NewModel(allBuildUpdates, "Building Services")) // blocks but quits once the above updates channel is closed by the build process _, err = prog.Run() tui.CheckErr(err) @@ -145,6 +150,16 @@ var runCmd = &cobra.Command{ } }() + go func() { + err := proj.RunBatches(localCloud, stopChan, updatesChan, loadEnv) + if err != nil { + localCloud.Stop() + + tui.CheckErr(err) + } + }() + + tui.CheckErr(err) // FIXME: This is a hack to get labelled logs into the TUI // We should refactor the system logs to be more generic systemChan := make(chan project.ServiceRunUpdate) diff --git a/cmd/stack.go b/cmd/stack.go index 2d4a1e2b8..4798d3cec 100644 --- a/cmd/stack.go +++ b/cmd/stack.go @@ -23,6 +23,7 @@ import ( "strings" "github.com/charmbracelet/lipgloss" + "github.com/samber/lo" "github.com/spf13/afero" "github.com/spf13/cobra" "google.golang.org/protobuf/types/known/structpb" @@ -198,6 +199,11 @@ var stackUpdateCmd = &cobra.Command{ buildUpdates, err := proj.BuildServices(fs) tui.CheckErr(err) + batchBuildUpdates, err := proj.BuildBatches(fs) + tui.CheckErr(err) + + allBuildUpdates := lo.FanIn(10, buildUpdates, batchBuildUpdates) + if isNonInteractive() { fmt.Println("building project services") for _, service := range proj.GetServices() { @@ -205,13 +211,13 @@ var stackUpdateCmd = &cobra.Command{ } // non-interactive environment - for update := range buildUpdates { + for update := range allBuildUpdates { for _, line := range strings.Split(strings.TrimSuffix(update.Message, "\n"), "\n") { fmt.Printf("%s [%s]: %s\n", update.ServiceName, update.Status, line) } } } else { - prog := teax.NewProgram(build.NewModel(buildUpdates, "Building Services")) + prog := teax.NewProgram(build.NewModel(allBuildUpdates, "Building Services")) // blocks but quits once the above updates channel is closed by the build process buildModel, err := prog.Run() tui.CheckErr(err) @@ -225,6 +231,9 @@ var stackUpdateCmd = &cobra.Command{ serviceRequirements, err := proj.CollectServicesRequirements() tui.CheckErr(err) + batchRequirements, err := proj.CollectBatchRequirements() + tui.CheckErr(err) + additionalEnvFiles := []string{} if envFile != "" { @@ -245,12 +254,7 @@ var stackUpdateCmd = &cobra.Command{ envVariables["NITRIC_BETA_PROVIDERS"] = "true" } - defaultImageName, ok := proj.DefaultMigrationImage(fs) - if !ok { - defaultImageName = "" - } - - migrationImageContexts, err := collector.GetMigrationImageBuildContexts(serviceRequirements, fs) + migrationImageContexts, err := collector.GetMigrationImageBuildContexts(serviceRequirements, batchRequirements, fs) tui.CheckErr(err) // Build images from contexts and provide updates on the builds @@ -277,7 +281,7 @@ var stackUpdateCmd = &cobra.Command{ } } - spec, err := collector.ServiceRequirementsToSpec(proj.Name, envVariables, serviceRequirements, defaultImageName) + spec, err := collector.ServiceRequirementsToSpec(proj.Name, envVariables, serviceRequirements, batchRequirements) tui.CheckErr(err) providerStdout := make(chan string) diff --git a/cmd/start.go b/cmd/start.go index 79cad9af9..16ac3bd49 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -223,6 +223,15 @@ var startCmd = &cobra.Command{ tui.CheckErr(err) } }() + // FIXME: Duplicate code + go func() { + err := proj.RunBatchesWithCommand(localCloud, stopChan, updatesChan, localEnv) + if err != nil { + localCloud.Stop() + + tui.CheckErr(err) + } + }() // FIXME: This is a hack to get labelled logs into the TUI // We should refactor the system logs to be more generic diff --git a/go.mod b/go.mod index 490b5e4ba..249221bbd 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.23.0 replace github.com/mattn/go-ieproxy => github.com/darthShadow/go-ieproxy v0.0.0-20220916090656-69928ad83ed6 -require github.com/golangci/golangci-lint v1.60.3 +require github.com/golangci/golangci-lint v1.61.0 require ( github.com/AlecAivazis/survey/v2 v2.3.6 @@ -23,14 +23,14 @@ require ( github.com/hashicorp/consul/sdk v0.13.0 github.com/hashicorp/go-getter v1.6.2 github.com/hashicorp/go-version v1.7.0 - github.com/nitrictech/nitric/core v0.0.0-20240827004051-cd5d36aaa8e6 + github.com/nitrictech/nitric/core v0.0.0-20241003062412-76ea6275fb0b github.com/pkg/errors v0.9.1 github.com/spf13/cobra v1.8.1 - github.com/valyala/fasthttp v1.51.0 - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/mod v0.20.0 // indirect + github.com/valyala/fasthttp v1.55.0 + golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e // indirect + golang.org/x/mod v0.21.0 // indirect golang.org/x/oauth2 v0.22.0 // indirect - google.golang.org/grpc v1.64.1 + google.golang.org/grpc v1.66.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -46,7 +46,7 @@ require ( github.com/jackc/pgx/v5 v5.6.0 github.com/joho/godotenv v1.5.1 github.com/mattn/go-isatty v0.0.20 - github.com/nitrictech/nitric/cloud/common v0.0.0-20231206014944-68e146f4f69a + github.com/nitrictech/nitric/cloud/common v0.0.0-20241003062412-76ea6275fb0b github.com/olahol/melody v1.1.3 github.com/robfig/cron/v3 v3.0.1 github.com/samber/lo v1.38.1 @@ -61,31 +61,31 @@ require ( require ( 4d63.com/gocheckcompilerdirectives v1.2.1 // indirect 4d63.com/gochecknoglobals v0.2.1 // indirect - cloud.google.com/go v0.115.0 // indirect - cloud.google.com/go/auth v0.8.1 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect + cloud.google.com/go v0.115.1 // indirect + cloud.google.com/go/auth v0.9.3 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect cloud.google.com/go/compute/metadata v0.5.0 // indirect - cloud.google.com/go/iam v1.1.12 // indirect - cloud.google.com/go/storage v1.41.0 // indirect + cloud.google.com/go/iam v1.2.0 // indirect + cloud.google.com/go/storage v1.43.0 // indirect github.com/4meepo/tagalign v1.3.4 // indirect - github.com/Abirdcfly/dupword v0.0.14 // indirect + github.com/Abirdcfly/dupword v0.1.1 // indirect github.com/Antonboom/errname v0.1.13 // indirect github.com/Antonboom/nilnil v0.1.9 // indirect github.com/Antonboom/testifylint v1.4.3 // indirect github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect - github.com/Crocmagnon/fatcontext v0.4.0 // indirect + github.com/Crocmagnon/fatcontext v0.5.2 // indirect github.com/DataDog/zstd v1.5.5 // indirect github.com/Djarvur/go-err113 v0.1.0 // indirect github.com/GaijinEntertainment/go-exhaustruct/v3 v3.3.0 // indirect - github.com/Masterminds/semver/v3 v3.2.1 // indirect - github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/Masterminds/semver/v3 v3.3.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect github.com/OpenPeeDeeP/depguard/v2 v2.2.0 // indirect github.com/Sereal/Sereal v0.0.0-20221130110801-16a4f76670cd // indirect github.com/alecthomas/go-check-sumtype v0.1.4 // indirect github.com/alexkohler/nakedret/v2 v2.0.4 // indirect github.com/alexkohler/prealloc v1.0.0 // indirect github.com/alingse/asasalint v0.0.11 // indirect - github.com/andybalholm/brotli v1.0.5 // indirect + github.com/andybalholm/brotli v1.1.0 // indirect github.com/ashanbrown/forbidigo v1.6.0 // indirect github.com/ashanbrown/makezero v1.1.1 // indirect github.com/atotto/clipboard v0.1.4 // indirect @@ -103,14 +103,13 @@ require ( github.com/butuzov/mirror v1.2.0 // indirect github.com/catenacyber/perfsprint v0.7.1 // indirect github.com/ccojocar/zxcvbn-go v1.0.2 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charithe/durationcheck v0.0.10 // indirect github.com/chavacava/garif v0.1.0 // indirect - github.com/ckaznocha/intrange v0.1.2 // indirect + github.com/ckaznocha/intrange v0.2.0 // indirect github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect - github.com/containerd/log v0.1.0 // indirect github.com/curioswitch/go-reassign v0.2.0 // indirect - github.com/daixiang0/gci v0.13.4 // indirect + github.com/daixiang0/gci v0.13.5 // indirect github.com/denis-tingaikin/go-header v0.5.0 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect @@ -134,7 +133,7 @@ require ( github.com/go-toolsmith/astp v1.1.0 // indirect github.com/go-toolsmith/strparse v1.1.0 // indirect github.com/go-toolsmith/typep v1.1.0 // indirect - github.com/go-viper/mapstructure/v2 v2.0.0 // indirect + github.com/go-viper/mapstructure/v2 v2.1.0 // indirect github.com/go-xmlfmt/xmlfmt v1.1.2 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/gofrs/flock v0.12.1 // indirect @@ -149,7 +148,7 @@ require ( github.com/golangci/revgrep v0.5.3 // indirect github.com/golangci/unconvert v0.0.0-20240309020433-c5143eacb3ed // indirect github.com/google/s2a-go v0.1.8 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.3 // indirect github.com/googleapis/gax-go/v2 v2.13.0 // indirect github.com/gordonklaus/ineffassign v0.1.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect @@ -176,7 +175,7 @@ require ( github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/kisielk/errcheck v1.7.0 // indirect github.com/kkHAIKE/contextcheck v1.1.5 // indirect - github.com/klauspost/compress v1.17.0 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/kulti/thelper v0.6.3 // indirect github.com/kunwardeep/paralleltest v1.0.10 // indirect github.com/kyoh86/exportloopref v0.1.11 // indirect @@ -203,7 +202,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/moricho/tparallel v0.3.2 // indirect - github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b // indirect + github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.15.2 // indirect @@ -213,30 +212,30 @@ require ( github.com/nunnatsa/ginkgolinter v0.16.2 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.0.2 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/perimeterx/marshmallow v1.1.4 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/polyfloyd/go-errorlint v1.6.0 // indirect github.com/prometheus/client_golang v1.14.0 // indirect - github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/client_model v0.6.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect - github.com/quasilyte/go-ruleguard v0.4.2 // indirect + github.com/quasilyte/go-ruleguard v0.4.3-0.20240823090925-0fe6f58b47b1 // indirect github.com/quasilyte/go-ruleguard/dsl v0.3.22 // indirect github.com/quasilyte/gogrep v0.5.0 // indirect github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect github.com/rivo/uniseg v0.4.4 // indirect - github.com/ryancurrah/gomodguard v1.3.3 // indirect + github.com/ryancurrah/gomodguard v1.3.5 // indirect github.com/ryanrolds/sqlclosecheck v0.5.1 // indirect github.com/sanposhiho/wastedassign/v2 v2.0.7 // indirect github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/sashamelentyev/interfacebloat v1.1.0 // indirect github.com/sashamelentyev/usestdlibvars v1.27.0 // indirect github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee // indirect - github.com/securego/gosec/v2 v2.20.1-0.20240822074752-ab3f6c1c83a0 // indirect + github.com/securego/gosec/v2 v2.21.2 // indirect github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/sivchari/containedctx v1.0.3 // indirect @@ -253,7 +252,7 @@ require ( github.com/stretchr/testify v1.9.0 // indirect github.com/subosito/gotenv v1.4.1 // indirect github.com/tdakkota/asciicheck v0.2.0 // indirect - github.com/tetafro/godot v1.4.16 // indirect + github.com/tetafro/godot v1.4.17 // indirect github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966 // indirect github.com/timonwong/loggercheck v0.9.4 // indirect github.com/tomarrell/wrapcheck/v2 v2.9.0 // indirect @@ -272,29 +271,29 @@ require ( go-simpler.org/musttag v0.12.2 // indirect go-simpler.org/sloglint v0.7.2 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect - go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect + go.opentelemetry.io/otel v1.29.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 // indirect - go.opentelemetry.io/otel/metric v1.28.0 // indirect - go.opentelemetry.io/otel/sdk v1.28.0 // indirect - go.opentelemetry.io/otel/trace v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.29.0 // indirect + go.opentelemetry.io/otel/sdk v1.29.0 // indirect + go.opentelemetry.io/otel/trace v1.29.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/automaxprocs v1.5.3 // indirect go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.26.0 // indirect + golang.org/x/crypto v0.27.0 // indirect golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f // indirect golang.org/x/net v0.28.0 // indirect - golang.org/x/sys v0.23.0 // indirect - golang.org/x/term v0.23.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/term v0.24.0 // indirect + golang.org/x/text v0.18.0 // indirect golang.org/x/time v0.6.0 // indirect golang.org/x/tools v0.24.0 // indirect - google.golang.org/api v0.192.0 // indirect - google.golang.org/genproto v0.0.0-20240730163845-b1a4ccb954bf // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240725223205-93522f1f2a9f // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf // indirect + google.golang.org/api v0.196.0 // indirect + google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect honnef.co/go/tools v0.5.1 // indirect mvdan.cc/gofumpt v0.7.0 // indirect diff --git a/go.sum b/go.sum index a6c5635c7..13015e43c 100644 --- a/go.sum +++ b/go.sum @@ -17,12 +17,12 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= -cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= -cloud.google.com/go/auth v0.8.1 h1:QZW9FjC5lZzN864p13YxvAtGUlQ+KgRL+8Sg45Z6vxo= -cloud.google.com/go/auth v0.8.1/go.mod h1:qGVp/Y3kDRSDZ5gFD/XPUfYQ9xW1iI7q8RIRoCyBbJc= -cloud.google.com/go/auth/oauth2adapt v0.2.3 h1:MlxF+Pd3OmSudg/b1yZ5lJwoXCEaeedAguodky1PcKI= -cloud.google.com/go/auth/oauth2adapt v0.2.3/go.mod h1:tMQXOfZzFuNuUxOypHlQEXgdfX5cuhwU+ffUuXRJE8I= +cloud.google.com/go v0.115.1 h1:Jo0SM9cQnSkYfp44+v+NQXHpcHqlnRJk2qxh6yvxxxQ= +cloud.google.com/go v0.115.1/go.mod h1:DuujITeaufu3gL68/lOFIirVNJwQeyf5UXyi+Wbgknc= +cloud.google.com/go/auth v0.9.3 h1:VOEUIAADkkLtyfr3BLa3R8Ed/j6w1jTBmARx+wb5w5U= +cloud.google.com/go/auth v0.9.3/go.mod h1:7z6VY+7h3KUdRov5F1i8NDP5ZzWKYmEPO842BgCsmTk= +cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= +cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -33,8 +33,10 @@ cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJ cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v1.1.12 h1:JixGLimRrNGcxvJEQ8+clfLxPlbeZA6MuRJ+qJNQ5Xw= -cloud.google.com/go/iam v1.1.12/go.mod h1:9LDX8J7dN5YRyzVHxwQzrQs9opFFqn0Mxs9nAeB+Hhg= +cloud.google.com/go/iam v1.2.0 h1:kZKMKVNk/IsSSc/udOb83K0hL/Yh/Gcqpz+oAkoIFN8= +cloud.google.com/go/iam v1.2.0/go.mod h1:zITGuWgsLZxd8OwAlX+eMFgZDXzBm7icj1PVTYG766Q= +cloud.google.com/go/longrunning v0.6.0 h1:mM1ZmaNsQsnb+5n1DNPeL0KwQd9jQRqSqSDEkBZr+aI= +cloud.google.com/go/longrunning v0.6.0/go.mod h1:uHzSZqW89h7/pasCWNYdUpwGz3PcVWhrWupreVPYLts= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -44,13 +46,13 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.41.0 h1:RusiwatSu6lHeEXe3kglxakAmAbfV+rhtPqA6i8RBx0= -cloud.google.com/go/storage v1.41.0/go.mod h1:J1WCa/Z2FcgdEDuPUY8DxT5I+d9mFKsCepp5vR6Sq80= +cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= +cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/4meepo/tagalign v1.3.4 h1:P51VcvBnf04YkHzjfclN6BbsopfJR5rxs1n+5zHt+w8= github.com/4meepo/tagalign v1.3.4/go.mod h1:M+pnkHH2vG8+qhE5bVc/zeP7HS/j910Fwa9TUSyZVI0= -github.com/Abirdcfly/dupword v0.0.14 h1:3U4ulkc8EUo+CaT105/GJ1BQwtgyj6+VaBVbAX11Ba8= -github.com/Abirdcfly/dupword v0.0.14/go.mod h1:VKDAbxdY8YbKUByLGg8EETzYSuC4crm9WwI6Y3S0cLI= +github.com/Abirdcfly/dupword v0.1.1 h1:Bsxe0fIw6OwBtXMIncaTxCLHYO5BB+3mcsR5E8VXloY= +github.com/Abirdcfly/dupword v0.1.1/go.mod h1:B49AcJdTYYkpd4HjgAcutNGG9HZ2JWwKunH9Y2BA6sM= github.com/AlecAivazis/survey/v2 v2.3.6 h1:NvTuVHISgTHEHeBFqt6BHOe4Ny/NwGZr7w+F8S9ziyw= github.com/AlecAivazis/survey/v2 v2.3.6/go.mod h1:4AuI9b7RjAR+G7v9+C4YSlX/YL3K3cWNXgWXOhllqvI= github.com/Antonboom/errname v0.1.13 h1:JHICqsewj/fNckzrfVSe+T33svwQxmjC+1ntDsHOVvM= @@ -65,18 +67,18 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/kWtOwnv/G+AzdKuy2ZrqINhenH4HyNs= github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Crocmagnon/fatcontext v0.4.0 h1:4ykozu23YHA0JB6+thiuEv7iT6xq995qS1vcuWZq0tg= -github.com/Crocmagnon/fatcontext v0.4.0/go.mod h1:ZtWrXkgyfsYPzS6K3O88va6t2GEglG93vnII/F94WC0= +github.com/Crocmagnon/fatcontext v0.5.2 h1:vhSEg8Gqng8awhPju2w7MKHqMlg4/NI+gSDHtR3xgwA= +github.com/Crocmagnon/fatcontext v0.5.2/go.mod h1:87XhRMaInHP44Q7Tlc7jkgKKB7kZAOPiDkFMdKCC+74= github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Djarvur/go-err113 v0.1.0 h1:uCRZZOdMQ0TZPHYTdYpoC0bLYJKPEHPUJ8MeAa51lNU= github.com/Djarvur/go-err113 v0.1.0/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= github.com/GaijinEntertainment/go-exhaustruct/v3 v3.3.0 h1:/fTUt5vmbkAcMBt4YQiuC23cV0kEsN1MVMNqeOW43cU= github.com/GaijinEntertainment/go-exhaustruct/v3 v3.3.0/go.mod h1:ONJg5sxcbsdQQ4pOW8TGdTidT2TMAUy/2Xhr8mrYaao= -github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= -github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= -github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= -github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= +github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= github.com/OpenPeeDeeP/depguard/v2 v2.2.0 h1:vDfG60vDtIuf0MEOhmLlLLSzqaRM8EMcgJPdp74zmpA= @@ -100,8 +102,8 @@ github.com/alexkohler/prealloc v1.0.0 h1:Hbq0/3fJPQhNkN0dR95AVrr6R7tou91y0uHG5pO github.com/alexkohler/prealloc v1.0.0/go.mod h1:VetnK3dIgFBBKmg0YnD9F9x6Icjd+9cvfHR56wJVlKE= github.com/alingse/asasalint v0.0.11 h1:SFwnQXJ49Kx/1GghOFz1XGqHYKp21Kq1nHad/0WQRnw= github.com/alingse/asasalint v0.0.11/go.mod h1:nCaoMhw7a9kSJObvQyVzNTPBDbNpdocqrSP7t/cW5+I= -github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= -github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM= github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII= github.com/asdine/storm v2.1.2+incompatible h1:dczuIkyqwY2LrtXPz8ixMrU/OFgZp71kbKTHGrXYt/Q= @@ -152,8 +154,8 @@ github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyY github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charithe/durationcheck v0.0.10 h1:wgw73BiocdBDQPik+zcEoBG/ob8uyBHf2iyoHGPf5w4= github.com/charithe/durationcheck v0.0.10/go.mod h1:bCWXb7gYRysD1CU3C+u4ceO49LoGOY1C1L6uouGNreQ= github.com/charmbracelet/bubbles v0.16.1 h1:6uzpAAaT9ZqKssntbvZMlksWHruQLNxg49H5WdeuYSY= @@ -168,8 +170,8 @@ github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXH github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/ckaznocha/intrange v0.1.2 h1:3Y4JAxcMntgb/wABQ6e8Q8leMd26JbX2790lIss9MTI= -github.com/ckaznocha/intrange v0.1.2/go.mod h1:RWffCw/vKBwHeOEwWdCikAtY0q4gGt8VhJZEEA5n+RE= +github.com/ckaznocha/intrange v0.2.0 h1:FykcZuJ8BD7oX93YbO1UY9oZtkRbp+1/kJcDjkefYLs= +github.com/ckaznocha/intrange v0.2.0/go.mod h1:r5I7nUlAAG56xmkOpw4XVr16BXhwYTUdcuRFeevn1oE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2wIvVRd/hEHD7lacgqrCPS+k8g1MndzfWY= @@ -181,8 +183,8 @@ github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI= github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/curioswitch/go-reassign v0.2.0 h1:G9UZyOcpk/d7Gd6mqYgd8XYWFMw/znxwGDUstnC9DIo= github.com/curioswitch/go-reassign v0.2.0/go.mod h1:x6OpXuWvgfQaMGks2BZybTngWjT84hqJfKoO8Tt/Roc= -github.com/daixiang0/gci v0.13.4 h1:61UGkmpoAcxHM2hhNkZEf5SzwQtWJXTSws7jaPyqwlw= -github.com/daixiang0/gci v0.13.4/go.mod h1:12etP2OniiIdP4q+kjUGrC/rUagga7ODbqsom5Eo5Yk= +github.com/daixiang0/gci v0.13.5 h1:kThgmH1yBmZSBCh1EJVxQ7JsHpm5Oms0AMed/0LaH4c= +github.com/daixiang0/gci v0.13.5/go.mod h1:12etP2OniiIdP4q+kjUGrC/rUagga7ODbqsom5Eo5Yk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -217,8 +219,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/firefart/nonamedreturns v1.0.5 h1:tM+Me2ZaXs8tfdDw3X6DOX++wMCOqzYUho6tUTYIdRA= github.com/firefart/nonamedreturns v1.0.5/go.mod h1:gHJjDqhGM4WyPt639SOZs+G89Ko7QKH5R5BhnO6xJhw= -github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= -github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= @@ -277,8 +279,8 @@ github.com/go-toolsmith/strparse v1.1.0 h1:GAioeZUK9TGxnLS+qfdqNbA4z0SSm5zVNtCQi github.com/go-toolsmith/strparse v1.1.0/go.mod h1:7ksGy58fsaQkGQlY8WVoBFNyEPMGuJin1rfoPS4lBSQ= github.com/go-toolsmith/typep v1.1.0 h1:fIRYDyF+JywLfqzyhdiHzRop/GQDxxNhLGQ6gFUNHus= github.com/go-toolsmith/typep v1.1.0/go.mod h1:fVIw+7zjdsMxDA3ITWnH1yOiw1rnTQKCsF/sk2H/qig= -github.com/go-viper/mapstructure/v2 v2.0.0 h1:dhn8MZ1gZ0mzeodTG3jt5Vj/o87xZKuNAprG2mQfMfc= -github.com/go-viper/mapstructure/v2 v2.0.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-viper/mapstructure/v2 v2.1.0 h1:gHnMa2Y/pIxElCH2GlZZ1lZSsn6XMtufpGyP1XxdC/w= +github.com/go-viper/mapstructure/v2 v2.1.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/go-xmlfmt/xmlfmt v1.1.2 h1:Nea7b4icn8s57fTx1M5AI4qQT5HEM3rVUO8MuE6g80U= github.com/go-xmlfmt/xmlfmt v1.1.2/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= @@ -329,8 +331,8 @@ github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9 github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk= github.com/golangci/gofmt v0.0.0-20240816233607-d8596aa466a9 h1:/1322Qns6BtQxUZDTAT4SdcoxknUki7IAoK4SAXr8ME= github.com/golangci/gofmt v0.0.0-20240816233607-d8596aa466a9/go.mod h1:Oesb/0uFAyWoaw1U1qS5zyjCg5NP9C9iwjnI4tIsXEE= -github.com/golangci/golangci-lint v1.60.3 h1:l38A5de24ZeDlcFF+EB7m3W5joPD99/hS5SIHJPyZa0= -github.com/golangci/golangci-lint v1.60.3/go.mod h1:J4vOpcjzRI+lDL2DKNGBZVB3EQSBfCBCMpaydWLtJNo= +github.com/golangci/golangci-lint v1.61.0 h1:VvbOLaRVWmyxCnUIMTbf1kDsaJbTzH20FAMXTAlQGu8= +github.com/golangci/golangci-lint v1.61.0/go.mod h1:e4lztIrJJgLPhWvFPDkhiMwEFRrWlmFbrZea3FsJyN8= github.com/golangci/misspell v0.6.0 h1:JCle2HUTNWirNlDIAUO44hUsKhOFqGPoC4LZxlaSXDs= github.com/golangci/misspell v0.6.0/go.mod h1:keMNyY6R9isGaSAu+4Q8NMBwMPkh15Gtc8UCVoDtAWo= github.com/golangci/modinfo v0.3.4 h1:oU5huX3fbxqQXdfspamej74DFX0kyGLkw1ppvXoJ8GA= @@ -371,16 +373,16 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQuYXQ+6EN5stWmeY/AZqtM8xk9k= -github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSFBy+X1V0o+l+8NF1avt4HWl7cA= +github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= -github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/enterprise-certificate-proxy v0.3.3 h1:QRje2j5GZimBzlbhGA2V2QlGNgL8G6e+wGo/+/2bWI0= +github.com/googleapis/enterprise-certificate-proxy v0.3.3/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= @@ -483,8 +485,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kkHAIKE/contextcheck v1.1.5 h1:CdnJh63tcDe53vG+RebdpdXJTc9atMgGqdx8LXxiilg= github.com/kkHAIKE/contextcheck v1.1.5/go.mod h1:O930cpht4xb1YQpK+1+AgoM3mFsvxr7uyFptcnWTYUA= github.com/klauspost/compress v1.11.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= -github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -579,8 +581,8 @@ github.com/moricho/tparallel v0.3.2 h1:odr8aZVFA3NZrNybggMkYO3rgPRcqjeQUlBBFVxKH github.com/moricho/tparallel v0.3.2/go.mod h1:OQ+K3b4Ln3l2TZveGCywybl68glfLEwFGqvnjok8b+U= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b h1:1XF24mVaiu7u+CFywTdcDo2ie1pzzhwjt6RHqzpMU34= -github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b/go.mod h1:fQuZ0gauxyBcmsdE3ZT4NasjaRdxmbCS0jRHsrWu3Ho= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= @@ -595,10 +597,10 @@ github.com/nishanths/exhaustive v0.12.0 h1:vIY9sALmw6T/yxiASewa4TQcFsVYZQQRUQJhK github.com/nishanths/exhaustive v0.12.0/go.mod h1:mEZ95wPIZW+x8kC4TgC+9YCUgiST7ecevsVDTgc2obs= github.com/nishanths/predeclared v0.2.2 h1:V2EPdZPliZymNAn79T8RkNApBjMmVKh5XRpLm/w98Vk= github.com/nishanths/predeclared v0.2.2/go.mod h1:RROzoN6TnGQupbC+lqggsOlcgysk3LMK/HI84Mp280c= -github.com/nitrictech/nitric/cloud/common v0.0.0-20231206014944-68e146f4f69a h1:BgsbgSm3iaObiUqGYCdX5mtavHqNJ/3FDxgYeBG+hpw= -github.com/nitrictech/nitric/cloud/common v0.0.0-20231206014944-68e146f4f69a/go.mod h1:+X62o2IvXWO1jw3758kdy+JWDCd5f1hCgqdPa4/Gtbo= -github.com/nitrictech/nitric/core v0.0.0-20240827004051-cd5d36aaa8e6 h1:4BCmJEWC4zX5dyTj60xenHF7FgE4iu4TtAGTINUiI38= -github.com/nitrictech/nitric/core v0.0.0-20240827004051-cd5d36aaa8e6/go.mod h1:N274XVBjYhGEQoT42baWM6/lETBQYQhqPpqUuk2gmLc= +github.com/nitrictech/nitric/cloud/common v0.0.0-20241003062412-76ea6275fb0b h1:wZeUrnmhYjdhSuL6ov+kVfuFJC9H14sk0kzEpt6aRoo= +github.com/nitrictech/nitric/cloud/common v0.0.0-20241003062412-76ea6275fb0b/go.mod h1:ZsCdb3xbukhXAp9ZNbV6qWJqRC+eLkxhXy8bhs/cC2A= +github.com/nitrictech/nitric/core v0.0.0-20241003062412-76ea6275fb0b h1:ImQFk66gRM3v9A6qmPImOiV3HJMDAX93X5rplMKn6ok= +github.com/nitrictech/nitric/core v0.0.0-20241003062412-76ea6275fb0b/go.mod h1:9bQnYPqLzq8CcPk5MHT3phg19CWJhDlFOfdIv27lwwM= github.com/nunnatsa/ginkgolinter v0.16.2 h1:8iLqHIZvN4fTLDC0Ke9tbSZVcyVHoBs0HIbnVSxfHJk= github.com/nunnatsa/ginkgolinter v0.16.2/go.mod h1:4tWRinDN1FeJgU+iJANW/kz7xKN5nYRAOfJDQUS9dOQ= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= @@ -609,14 +611,14 @@ github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.20.0 h1:PE84V2mHqoT1sglvHc8ZdQtPcwmvvt29WLEEO3xmdZw= -github.com/onsi/ginkgo/v2 v2.20.0/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI= -github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= -github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= +github.com/onsi/ginkgo/v2 v2.20.2 h1:7NVCeyIWROIAheY21RLS+3j2bb52W0W82tkberYytp4= +github.com/onsi/ginkgo/v2 v2.20.2/go.mod h1:K9gyxPIlb+aIvnZ8bd9Ak+YP18w3APlR+5coaZoE2ag= +github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8= +github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= -github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= @@ -626,8 +628,8 @@ github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT9 github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= -github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/perimeterx/marshmallow v1.1.4 h1:pZLDH9RjlLGGorbXhcaQLhfuV0pFMNfPO55FuFkxqLw= github.com/perimeterx/marshmallow v1.1.4/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -652,8 +654,8 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1: github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= +github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= @@ -667,8 +669,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/quasilyte/go-ruleguard v0.4.2 h1:htXcXDK6/rO12kiTHKfHuqR4kr3Y4M0J0rOL6CH/BYs= -github.com/quasilyte/go-ruleguard v0.4.2/go.mod h1:GJLgqsLeo4qgavUoL8JeGFNS7qcisx3awV/w9eWTmNI= +github.com/quasilyte/go-ruleguard v0.4.3-0.20240823090925-0fe6f58b47b1 h1:+Wl/0aFp0hpuHM3H//KMft64WQ1yX9LdJY64Qm/gFCo= +github.com/quasilyte/go-ruleguard v0.4.3-0.20240823090925-0fe6f58b47b1/go.mod h1:GJLgqsLeo4qgavUoL8JeGFNS7qcisx3awV/w9eWTmNI= github.com/quasilyte/go-ruleguard/dsl v0.3.22 h1:wd8zkOhSNr+I+8Qeciml08ivDt1pSXe60+5DqOpCjPE= github.com/quasilyte/go-ruleguard/dsl v0.3.22/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= github.com/quasilyte/gogrep v0.5.0 h1:eTKODPXbI8ffJMN+W2aE0+oL0z/nh8/5eNdiO34SOAo= @@ -687,8 +689,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryancurrah/gomodguard v1.3.3 h1:eiSQdJVNr9KTNxY2Niij8UReSwR8Xrte3exBrAZfqpg= -github.com/ryancurrah/gomodguard v1.3.3/go.mod h1:rsKQjj4l3LXe8N344Ow7agAy5p9yjsWOtRzUMYmA0QY= +github.com/ryancurrah/gomodguard v1.3.5 h1:cShyguSwUEeC0jS7ylOiG/idnd1TpJ1LfHGpV3oJmPU= +github.com/ryancurrah/gomodguard v1.3.5/go.mod h1:MXlEPQRxgfPQa62O8wzK3Ozbkv9Rkqr+wKjSxTdsNJE= github.com/ryanrolds/sqlclosecheck v0.5.1 h1:dibWW826u0P8jNLsLN+En7+RqWWTYrjCB9fJfSfdyCU= github.com/ryanrolds/sqlclosecheck v0.5.1/go.mod h1:2g3dUjoS6AL4huFdv6wn55WpLIDjY7ZgUR4J8HOO/XQ= github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= @@ -703,8 +705,8 @@ github.com/sashamelentyev/usestdlibvars v1.27.0 h1:t/3jZpSXtRPRf2xr0m63i32Zrusyu github.com/sashamelentyev/usestdlibvars v1.27.0/go.mod h1:9nl0jgOfHKWNFS43Ojw0i7aRoS4j6EBye3YBhmAIRF8= github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee h1:8Iv5m6xEo1NR1AvpV+7XmhI4r39LGNzwUL4YpMuL5vk= github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee/go.mod h1:qwtSXrKuJh/zsFQ12yEE89xfCrGKK63Rr7ctU/uCo4g= -github.com/securego/gosec/v2 v2.20.1-0.20240822074752-ab3f6c1c83a0 h1:VqD4JMoqwuuCz8GZlBDsIDyE6K4YUsWJpbNtuOWHoFk= -github.com/securego/gosec/v2 v2.20.1-0.20240822074752-ab3f6c1c83a0/go.mod h1:iyeMMRw8QEmueUSZ2VqmkQMiDyDcobfPnG00CV/NWdE= +github.com/securego/gosec/v2 v2.21.2 h1:deZp5zmYf3TWwU7A7cR2+SolbTpZ3HQiwFqnzQyEl3M= +github.com/securego/gosec/v2 v2.21.2/go.mod h1:au33kg78rNseF5PwPnTWhuYBFf534bvJRvOrgZ/bFzU= github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c h1:W65qqJCIOVP4jpqPQ0YvHYKwcMEMVWIzWC5iNQQfBTU= github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs= github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= @@ -764,8 +766,8 @@ github.com/tenntenn/modver v1.0.1 h1:2klLppGhDgzJrScMpkj9Ujy3rXPUspSjAcev9tSEBgA github.com/tenntenn/modver v1.0.1/go.mod h1:bePIyQPb7UeioSRkw3Q0XeMhYZSMx9B8ePqg6SAMGH0= github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3 h1:f+jULpRQGxTSkNYKJ51yaw6ChIqO+Je8UqsTKN/cDag= github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3/go.mod h1:ON8b8w4BN/kE1EOhwT0o+d62W65a6aPw1nouo9LMgyY= -github.com/tetafro/godot v1.4.16 h1:4ChfhveiNLk4NveAZ9Pu2AN8QZ2nkUGFuadM9lrr5D0= -github.com/tetafro/godot v1.4.16/go.mod h1:2oVxTBSftRTh4+MVfUaUXR6bn2GDXCaMcOG4Dk3rfio= +github.com/tetafro/godot v1.4.17 h1:pGzu+Ye7ZUEFx7LHU0dAKmCOXWsPjl7qA6iMGndsjPs= +github.com/tetafro/godot v1.4.17/go.mod h1:2oVxTBSftRTh4+MVfUaUXR6bn2GDXCaMcOG4Dk3rfio= github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966 h1:quvGphlmUVU+nhpFa4gg4yJyTRJ13reZMDHrKwYw53M= github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966/go.mod h1:27bSVNWSBOHm+qRp1T9qzaIpsWEP6TbUnei/43HK+PQ= github.com/timonwong/loggercheck v0.9.4 h1:HKKhqrjcVj8sxL7K77beXh0adEm6DLjV/QOGeMXEVi4= @@ -789,8 +791,8 @@ github.com/uudashr/gocognit v1.1.3 h1:l+a111VcDbKfynh+airAy/DJQKaXh2m9vkoysMPSZy github.com/uudashr/gocognit v1.1.3/go.mod h1:aKH8/e8xbTRBwjbCkwZ8qt4l2EpKXl31KMHgSS+lZ2U= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA= -github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g= +github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8= +github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= @@ -827,22 +829,22 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0 h1:A3SayB3rNyt+1S6qpI9mHPkeHTZbD7XILEqWnYZb2l0= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0/go.mod h1:27iA5uvhuRNmalO+iEUdVn5ZMj2qy10Mm+XRIpRmyuU= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 h1:Xs2Ncz0gNihqu9iosIZ5SkBbWo5T8JhhLJFMQL1qmLI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0/go.mod h1:vy+2G/6NvVMpwGX/NyLqcC41fxepnuKHk16E6IZUcJc= -go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= -go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= +go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= +go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 h1:j9+03ymgYhPKmeXGk5Zu+cIZOlVzd9Zv7QIiyItjFBU= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0/go.mod h1:Y5+XiUG4Emn1hTfciPzGPJaSI+RpDts6BnCIir0SLqk= -go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= -go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= -go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= -go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= -go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= -go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= +go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= +go.opentelemetry.io/otel/sdk v1.29.0 h1:vkqKjk7gwhS8VaWb0POZKmIEDimRCMsopNYnriHyryo= +go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok= +go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= +go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -850,8 +852,8 @@ go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= -go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= @@ -864,8 +866,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -876,8 +878,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e h1:I88y4caeGeuDQxgdoFPUq097j7kNfw6uvuiNxUBfcBk= +golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/exp/typeparams v0.0.0-20230203172020-98cc5a0785f9/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f h1:phY1HzDcf18Aq9A8KkmRtY9WvOFIxN8wgfvy6Zm1DV8= @@ -910,8 +912,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= -golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1040,8 +1042,8 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1049,8 +1051,8 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1061,8 +1063,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1136,8 +1138,6 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1154,8 +1154,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.192.0 h1:PljqpNAfZaaSpS+TnANfnNAXKdzHM/B9bKhwRlo7JP0= -google.golang.org/api v0.192.0/go.mod h1:9VcphjvAxPKLmSxVSzPlSRXy/5ARMEw5bf58WoVXafQ= +google.golang.org/api v0.196.0 h1:k/RafYqebaIJBO3+SMnfEGtFVlvp5vSgqTUF54UN/zg= +google.golang.org/api v0.196.0/go.mod h1:g9IL21uGkYgvQ5BZg6BAtoGJQIm8r6EgaAbpNey5wBE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1193,12 +1193,12 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20240730163845-b1a4ccb954bf h1:OqdXDEakZCVtDiZTjcxfwbHPCT11ycCEsTKesBVKvyY= -google.golang.org/genproto v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:mCr1K1c8kX+1iSBREvU3Juo11CB+QOEWxbRS01wWl5M= -google.golang.org/genproto/googleapis/api v0.0.0-20240725223205-93522f1f2a9f h1:b1Ln/PG8orm0SsBbHZWke8dDp2lrCD4jSmfglFpTZbk= -google.golang.org/genproto/googleapis/api v0.0.0-20240725223205-93522f1f2a9f/go.mod h1:AHT0dDg3SoMOgZGnZk29b5xTbPHMoEC8qthmBLJCpys= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf h1:liao9UHurZLtiEwBgT9LMOnKYsHze6eA6w1KQCMVN2Q= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 h1:BulPr26Jqjnd4eYDVe+YvyR7Yc2vJGkO5/0UxD0/jZU= +google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:hL97c3SYopEHblzpxRL4lSs523++l8DYxGM1FQiYmb4= +google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed h1:3RgNmBoI9MZhsj3QxC+AP/qQhNwpCLOvYDYYsFrhFt0= +google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1212,8 +1212,8 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= -google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1253,8 +1253,8 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= -gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/cloud/batch/batch.go b/pkg/cloud/batch/batch.go new file mode 100644 index 000000000..d1a94a883 --- /dev/null +++ b/pkg/cloud/batch/batch.go @@ -0,0 +1,198 @@ +// Copyright Nitric Pty Ltd. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package batch + +import ( + "context" + "fmt" + "maps" + "sync" + + "github.com/asaskevich/EventBus" + + "github.com/nitrictech/cli/pkg/grpcx" + "github.com/nitrictech/nitric/core/pkg/logger" + batchpb "github.com/nitrictech/nitric/core/pkg/proto/batch/v1" + "github.com/nitrictech/nitric/core/pkg/workers/jobs" +) + +type BatchRunner func(req *batchpb.JobSubmitRequest) error + +type ( + jobName = string + serviceName = string +) + +type ActionState struct { + JobName string + Payload string + Success bool +} + +type ( + State = map[jobName]map[serviceName]int + LocalBatchService struct { + *jobs.JobManager + batchpb.UnimplementedBatchServer + + state State + batchLock sync.RWMutex + + bus EventBus.Bus + } +) + +var ( + _ batchpb.BatchServer = (*LocalBatchService)(nil) + _ batchpb.JobServer = (*LocalBatchService)(nil) +) + +const ( + localBatchTopic = "local-batch" + localBatchDeliveryTopic = "local-batch-delivery" +) + +func (l *LocalBatchService) SubscribeToState(subscriberFunction func(State)) { + // ignore the error, it's only returned if the fn param isn't a function + _ = l.bus.Subscribe(localBatchTopic, subscriberFunction) +} + +func (l *LocalBatchService) publishState() { + l.bus.Publish(localBatchTopic, l.GetState()) +} + +func (l *LocalBatchService) GetState() State { + return maps.Clone(l.state) +} + +func (l *LocalBatchService) publishAction(action ActionState) { + l.bus.Publish(localBatchDeliveryTopic, action) +} + +func (l *LocalBatchService) SubscribeToAction(subscription func(ActionState)) { + // ignore the error, it's only returned if the fn param isn't a function + _ = l.bus.Subscribe(localBatchDeliveryTopic, subscription) +} + +func (l *LocalBatchService) registerJob(serviceName string, registration *batchpb.RegistrationRequest) { + l.batchLock.Lock() + defer l.batchLock.Unlock() + + if l.state[registration.JobName] == nil { + l.state[registration.JobName] = make(map[string]int) + } + + l.state[registration.JobName][serviceName]++ + + l.publishState() +} + +func (l *LocalBatchService) unregisterJob(serviceName string, registration *batchpb.RegistrationRequest) { + l.batchLock.Lock() + defer l.batchLock.Unlock() + + if l.state[registration.JobName] == nil { + l.state[registration.JobName] = make(map[string]int) + } + + l.state[registration.JobName][serviceName]-- + + if l.state[registration.JobName][serviceName] == 0 { + delete(l.state, registration.JobName) + } + + l.publishState() +} + +func (l *LocalBatchService) HandleJob(stream batchpb.Job_HandleJobServer) error { + serviceName, err := grpcx.GetServiceNameFromStream(stream) + if err != nil { + return err + } + + peekableStream := grpcx.NewPeekableStreamServer[*batchpb.ServerMessage, *batchpb.ClientMessage](stream) + + firstRequest, err := peekableStream.Peek() + if err != nil { + return err + } + + if firstRequest.GetRegistrationRequest() == nil { + return fmt.Errorf("first request must be a registration request") + } + + err = stream.Send(&batchpb.ServerMessage{ + Id: firstRequest.Id, + Content: &batchpb.ServerMessage_RegistrationResponse{ + RegistrationResponse: &batchpb.RegistrationResponse{}, + }, + }) + if err != nil { + return err + } + + // Keep track of our local batch subscriptions + l.registerJob(serviceName, firstRequest.GetRegistrationRequest()) + defer l.unregisterJob(serviceName, firstRequest.GetRegistrationRequest()) + + return l.JobManager.HandleJob(peekableStream) +} + +func (l *LocalBatchService) SubmitJob(ctx context.Context, req *batchpb.JobSubmitRequest) (*batchpb.JobSubmitResponse, error) { + go func() { + json, err := req.Data.GetStruct().MarshalJSON() + if err != nil { + logger.Errorf("Error marshalling job request data: %s", err.Error()) + } + + _, err = l.HandleJobRequest(&batchpb.ServerMessage{ + Content: &batchpb.ServerMessage_JobRequest{ + JobRequest: &batchpb.JobRequest{ + JobName: req.GetJobName(), + Data: req.Data, + }, + }, + }) + if err != nil { + logger.Errorf("Error handling job request: %s", err.Error()) + + l.publishAction(ActionState{ + JobName: req.GetJobName(), + Success: false, + Payload: string(json), + }) + + return + } + + l.publishAction(ActionState{ + JobName: req.GetJobName(), + Success: true, + Payload: string(json), + }) + }() + + return &batchpb.JobSubmitResponse{}, nil +} + +func NewLocalBatchService() *LocalBatchService { + return &LocalBatchService{ + JobManager: jobs.New(), + state: make(map[string]map[string]int), + bus: EventBus.New(), + } +} diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index b04839f29..5352b424b 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -25,6 +25,7 @@ import ( "google.golang.org/grpc/reflection" "github.com/nitrictech/cli/pkg/cloud/apis" + "github.com/nitrictech/cli/pkg/cloud/batch" "github.com/nitrictech/cli/pkg/cloud/gateway" "github.com/nitrictech/cli/pkg/cloud/http" "github.com/nitrictech/cli/pkg/cloud/keyvalue" @@ -55,6 +56,7 @@ type LocalCloud struct { servers map[ServiceName]*server.NitricServer Apis *apis.LocalApiGatewayService + Batch *batch.LocalBatchService KeyValue *keyvalue.BoltDocService Gateway *gateway.LocalGatewayService Http *http.LocalHttpProxy @@ -66,8 +68,6 @@ type LocalCloud struct { Websockets *websockets.LocalWebsocketService Queues *queues.LocalQueuesService Databases *sql.LocalSqlServer - - // Store all the plugins locally } // StartLocalNitric - starts the Nitric Server, including plugins and their local dependencies (e.g. local versions of cloud services) @@ -87,6 +87,75 @@ func (lc *LocalCloud) Stop() { } } +func (lc *LocalCloud) AddBatch(batchName string) (int, error) { + lc.serverLock.Lock() + defer lc.serverLock.Unlock() + + if _, ok := lc.servers[batchName]; ok { + return 0, fmt.Errorf("batch %s already added", batchName) + } + + // get an available port + ports, err := netx.TakePort(1) + if err != nil { + return 0, err + } + + nitricRuntimeServer, _ := server.New( + server.WithJobHandlerPlugin(lc.Batch), + server.WithBatchPlugin(lc.Batch), + server.WithResourcesPlugin(lc.Resources), + server.WithApiPlugin(lc.Apis), + server.WithHttpPlugin(lc.Http), + server.WithSqlPlugin(lc.Databases), + server.WithServiceAddress(fmt.Sprintf("0.0.0.0:%d", ports[0])), + server.WithSecretManagerPlugin(lc.Secrets), + server.WithStoragePlugin(lc.Storage), + server.WithKeyValuePlugin(lc.KeyValue), + server.WithGatewayPlugin(lc.Gateway), + server.WithWebsocketPlugin(lc.Websockets), + server.WithQueuesPlugin(lc.Queues), + server.WithMinWorkers(0), + server.WithChildCommand([]string{})) + + // Create a watcher that clears old resources when the service is restarted + _, err = resources.NewServiceResourceRefresher(batchName, resources.NewServiceResourceRefresherArgs{ + Resources: lc.Resources, + Apis: lc.Apis, + Schedules: lc.Schedules, + Http: lc.Http, + Listeners: lc.Storage, + Websockets: lc.Websockets, + Topics: lc.Topics, + Storage: lc.Storage, + BatchJobs: lc.Batch, + }) + if err != nil { + return 0, err + } + + go func() { + interceptor, streamInterceptor := grpcx.CreateServiceNameInterceptor(batchName) + + srv := grpc.NewServer( + grpc.UnaryInterceptor(interceptor), + grpc.StreamInterceptor(streamInterceptor), + ) + + // Enable reflection on the gRPC server for local testing + reflection.Register(srv) + + err := nitricRuntimeServer.Start(server.WithGrpcServer(srv)) + if err != nil { + logger.Errorf("Error starting nitric server: %s", err.Error()) + } + }() + + lc.servers[batchName] = nitricRuntimeServer + + return ports[0], nil +} + func (lc *LocalCloud) AddService(serviceName string) (int, error) { lc.serverLock.Lock() defer lc.serverLock.Unlock() @@ -102,6 +171,7 @@ func (lc *LocalCloud) AddService(serviceName string) (int, error) { } nitricRuntimeServer, _ := server.New( + server.WithBatchPlugin(lc.Batch), server.WithResourcesPlugin(lc.Resources), server.WithApiPlugin(lc.Apis), server.WithHttpPlugin(lc.Http), @@ -131,6 +201,7 @@ func (lc *LocalCloud) AddService(serviceName string) (int, error) { Websockets: lc.Websockets, Topics: lc.Topics, Storage: lc.Storage, + BatchJobs: lc.Batch, }) if err != nil { return 0, err @@ -185,6 +256,7 @@ func New(projectName string, opts LocalCloudOptions) (*LocalCloud, error) { } localApis := apis.NewLocalApiGatewayService() + localBatch := batch.NewLocalBatchService() localSchedules := schedules.NewLocalSchedulesService() localHttpProxy := http.NewLocalHttpProxyService() @@ -202,6 +274,7 @@ func New(projectName string, opts LocalCloudOptions) (*LocalCloud, error) { TLSCredentials: opts.TLSCredentials, LogWriter: opts.LogWriter, LocalConfig: opts.LocalConfig, + BatchPlugin: localBatch, }) if err != nil { return nil, err @@ -229,6 +302,7 @@ func New(projectName string, opts LocalCloudOptions) (*LocalCloud, error) { return &LocalCloud{ servers: make(map[string]*server.NitricServer), Apis: localApis, + Batch: localBatch, Http: localHttpProxy, Resources: localResources, Schedules: localSchedules, diff --git a/pkg/cloud/gateway/gateway.go b/pkg/cloud/gateway/gateway.go index 3902992f0..c8fd9684d 100644 --- a/pkg/cloud/gateway/gateway.go +++ b/pkg/cloud/gateway/gateway.go @@ -38,6 +38,7 @@ import ( "google.golang.org/protobuf/types/known/structpb" "github.com/nitrictech/cli/pkg/cloud/apis" + "github.com/nitrictech/cli/pkg/cloud/batch" "github.com/nitrictech/cli/pkg/cloud/http" "github.com/nitrictech/cli/pkg/cloud/schedules" "github.com/nitrictech/cli/pkg/cloud/topics" @@ -51,6 +52,7 @@ import ( "github.com/nitrictech/nitric/core/pkg/gateway" apispb "github.com/nitrictech/nitric/core/pkg/proto/apis/v1" + batchpb "github.com/nitrictech/nitric/core/pkg/proto/batch/v1" schedulespb "github.com/nitrictech/nitric/core/pkg/proto/schedules/v1" topicspb "github.com/nitrictech/nitric/core/pkg/proto/topics/v1" websocketspb "github.com/nitrictech/nitric/core/pkg/proto/websockets/v1" @@ -91,6 +93,7 @@ type LocalGatewayService struct { websocketPlugin *websockets.LocalWebsocketService topicsPlugin *topics.LocalTopicsAndSubscribersService schedulesPlugin *schedules.LocalSchedulesService + batchPlugin *batch.LocalBatchService serviceListener net.Listener localConfig localconfig.LocalConfiguration @@ -456,6 +459,38 @@ func (s *LocalGatewayService) handleSchedulesTrigger(ctx *fasthttp.RequestCtx) { ctx.SuccessString("text/plain", "Successfully triggered schedule") } +func (s *LocalGatewayService) handleBatchJobTrigger(ctx *fasthttp.RequestCtx) { + jobName := ctx.UserValue("name").(string) + + // Get the incoming data as JobData_Struct + payload := map[string]interface{}{} + + err := json.Unmarshal(ctx.Request.Body(), &payload) + if err != nil { + ctx.Error(fmt.Sprintf("Error parsing JSON: %v", err), 400) + return + } + + st, err := structpb.NewStruct(payload) + if err != nil { + ctx.Error(fmt.Sprintf("Error serializing job message from payload: %v", err), 400) + return + } + + jobSubmitRequest := &batchpb.JobSubmitRequest{ + JobName: jobName, + Data: &batchpb.JobData{Data: &batchpb.JobData_Struct{Struct: st}}, + } + + _, err = s.batchPlugin.SubmitJob(context.Background(), jobSubmitRequest) + if err != nil { + ctx.Error(fmt.Sprintf("Error handling batch job trigger: %v", err), 500) + return + } + + ctx.SuccessString("text/plain", "Successfully triggered job") +} + func (s *LocalGatewayService) refreshApis(apiState apis.State) { s.lock.Lock() defer s.lock.Unlock() @@ -704,6 +739,7 @@ const nameParam = "{name}" const ( topicPath = "/topics/" + nameParam schedulePath = "/schedules/" + nameParam + batchPath = "/jobs/" + nameParam ) func (s *LocalGatewayService) GetTopicTriggerUrl(topicName string) string { @@ -717,6 +753,11 @@ func (s *LocalGatewayService) GetScheduleManualTriggerUrl(scheduleName string) s return endpoint } +func (s *LocalGatewayService) GetBatchTriggerUrl(jobName string) string { + endpoint, _ := url.JoinPath("http://"+s.GetTriggerAddress(), strings.Replace(batchPath, nameParam, jobName, 1)) + return endpoint +} + func (s *LocalGatewayService) Start(opts *gateway.GatewayStartOpts) error { var err error // Assign the pool and block @@ -728,6 +769,7 @@ func (s *LocalGatewayService) Start(opts *gateway.GatewayStartOpts) error { // Publish to a topic r.POST(topicPath, s.handleTopicRequest) r.POST(schedulePath, s.handleSchedulesTrigger) + r.POST(batchPath, s.handleBatchJobTrigger) s.serviceServer = &fasthttp.Server{ ReadTimeout: time.Second * 1, @@ -811,6 +853,7 @@ type NewGatewayOpts struct { TLSCredentials *TLSCredentials LogWriter io.Writer LocalConfig localconfig.LocalConfiguration + BatchPlugin *batch.LocalBatchService } // Create new HTTP gateway @@ -821,5 +864,6 @@ func NewGateway(opts NewGatewayOpts) (*LocalGatewayService, error) { bus: EventBus.New(), logWriter: opts.LogWriter, localConfig: opts.LocalConfig, + batchPlugin: opts.BatchPlugin, }, nil } diff --git a/pkg/cloud/resources/resources.go b/pkg/cloud/resources/resources.go index 096cb55a2..92c7b78d4 100644 --- a/pkg/cloud/resources/resources.go +++ b/pkg/cloud/resources/resources.go @@ -33,6 +33,7 @@ type ResourceName = string type LocalResourcesState struct { Buckets *ResourceRegistrar[resourcespb.BucketResource] + BatchJobs *ResourceRegistrar[resourcespb.JobResource] KeyValueStores *ResourceRegistrar[resourcespb.KeyValueStoreResource] Policies *ResourceRegistrar[resourcespb.PolicyResource] Secrets *ResourceRegistrar[resourcespb.SecretResource] @@ -98,6 +99,9 @@ func (l *LocalResourcesService) Declare(ctx context.Context, req *resourcespb.Re } err = l.state.Policies.Register(policyName, serviceName, req.GetPolicy()) + + case resourcespb.ResourceType_Job: + err = l.state.BatchJobs.Register(req.Id.Name, serviceName, req.GetJob()) case resourcespb.ResourceType_Secret: err = l.state.Secrets.Register(req.Id.Name, serviceName, req.GetSecret()) case resourcespb.ResourceType_Topic: @@ -128,11 +132,14 @@ func (l *LocalResourcesService) ClearServiceResources(serviceName string) { l.state.Topics.ClearRequestingService(serviceName) l.state.Queues.ClearRequestingService(serviceName) l.state.ApiSecurityDefinitions.ClearRequestingService(serviceName) + l.state.SqlDatabases.ClearRequestingService(serviceName) + l.state.BatchJobs.ClearRequestingService(serviceName) } func NewLocalResourcesService(opts LocalResourcesOptions) *LocalResourcesService { return &LocalResourcesService{ state: LocalResourcesState{ + BatchJobs: NewResourceRegistrar[resourcespb.JobResource](), Buckets: NewResourceRegistrar[resourcespb.BucketResource](), KeyValueStores: NewResourceRegistrar[resourcespb.KeyValueStoreResource](), Policies: NewResourceRegistrar[resourcespb.PolicyResource](), diff --git a/pkg/cloud/resources/services.go b/pkg/cloud/resources/services.go index 99b4cf9fc..9244739e4 100644 --- a/pkg/cloud/resources/services.go +++ b/pkg/cloud/resources/services.go @@ -21,6 +21,7 @@ import ( "sync" "github.com/nitrictech/cli/pkg/cloud/apis" + "github.com/nitrictech/cli/pkg/cloud/batch" "github.com/nitrictech/cli/pkg/cloud/http" "github.com/nitrictech/cli/pkg/cloud/schedules" "github.com/nitrictech/cli/pkg/cloud/storage" @@ -36,6 +37,7 @@ type ServiceResourceRefresher struct { lock sync.RWMutex apiWorkers int + batchWorkers int scheduleWorkers int httpWorkers int listenerWorkers int @@ -45,6 +47,7 @@ type ServiceResourceRefresher struct { type UpdateArgs struct { apiState apis.State + batchState batch.State schedulesState schedules.State websocketState websockets.State bucketListenersState storage.State @@ -53,7 +56,7 @@ type UpdateArgs struct { } func (s *ServiceResourceRefresher) allWorkerCount() int { - return s.apiWorkers + s.scheduleWorkers + s.httpWorkers + s.listenerWorkers + s.subscriberWorkers + s.websocketWorkers + return s.apiWorkers + s.scheduleWorkers + s.httpWorkers + s.listenerWorkers + s.subscriberWorkers + s.websocketWorkers + s.batchWorkers } func (s *ServiceResourceRefresher) updatesWorkers(update UpdateArgs) { @@ -107,6 +110,13 @@ func (s *ServiceResourceRefresher) updatesWorkers(update UpdateArgs) { } } + if update.batchState != nil { + s.batchWorkers = 0 + for _, batch := range update.batchState { + s.batchWorkers += batch[s.serviceName] + } + } + // When the worker count for a service is 0, we can assume that the service is not running. // Typically this happens during a hot-reload/restarting a service and means the policies should be reset, since new policy requests will be coming in. if previous > 0 && s.allWorkerCount() == 0 { @@ -124,10 +134,11 @@ type NewServiceResourceRefresherArgs struct { Websockets *websockets.LocalWebsocketService Topics *topics.LocalTopicsAndSubscribersService Storage *storage.LocalStorageService + BatchJobs *batch.LocalBatchService } func NewServiceResourceRefresher(serviceName string, args NewServiceResourceRefresherArgs) (*ServiceResourceRefresher, error) { - if args.Resources == nil || args.Apis == nil || args.Schedules == nil || args.Http == nil || args.Listeners == nil || args.Websockets == nil { + if args.Resources == nil || args.Apis == nil || args.Schedules == nil || args.Http == nil || args.Listeners == nil || args.Websockets == nil || args.BatchJobs == nil { return nil, fmt.Errorf("all service plugins are required") } @@ -144,6 +155,12 @@ func NewServiceResourceRefresher(serviceName string, args NewServiceResourceRefr }) }) + args.BatchJobs.SubscribeToState(func(s batch.State) { + serviceState.updatesWorkers(UpdateArgs{ + batchState: s, + }) + }) + args.Http.SubscribeToState(func(s http.State) { serviceState.updatesWorkers(UpdateArgs{ httpState: s, diff --git a/pkg/cloud/schedules/schedules.go b/pkg/cloud/schedules/schedules.go index ed298343b..ff5509e92 100644 --- a/pkg/cloud/schedules/schedules.go +++ b/pkg/cloud/schedules/schedules.go @@ -49,6 +49,7 @@ type ActionState struct { ScheduleName string Success bool } + type LocalSchedulesService struct { *schedules.ScheduleWorkerManager cron *cron.Cron diff --git a/pkg/collector/batch.go b/pkg/collector/batch.go new file mode 100644 index 000000000..caa0c8cfa --- /dev/null +++ b/pkg/collector/batch.go @@ -0,0 +1,278 @@ +// Copyright Nitric Pty Ltd. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + "errors" + "fmt" + "sync" + + "google.golang.org/grpc" + + "github.com/nitrictech/cli/pkg/view/tui/components/view" + apispb "github.com/nitrictech/nitric/core/pkg/proto/apis/v1" + batchpb "github.com/nitrictech/nitric/core/pkg/proto/batch/v1" + httppb "github.com/nitrictech/nitric/core/pkg/proto/http/v1" + resourcespb "github.com/nitrictech/nitric/core/pkg/proto/resources/v1" + schedulespb "github.com/nitrictech/nitric/core/pkg/proto/schedules/v1" + storagepb "github.com/nitrictech/nitric/core/pkg/proto/storage/v1" + topicspb "github.com/nitrictech/nitric/core/pkg/proto/topics/v1" + websocketspb "github.com/nitrictech/nitric/core/pkg/proto/websockets/v1" +) + +type BatchRequirements struct { + batchName string + batchFile string + + resourceLock sync.Mutex + + buckets map[string]*resourcespb.BucketResource + keyValueStores map[string]*resourcespb.KeyValueStoreResource + topics map[string]*resourcespb.TopicResource + queues map[string]*resourcespb.QueueResource + sqlDatabases map[string]*resourcespb.SqlDatabaseResource + secrets map[string]*resourcespb.SecretResource + + jobs map[string]*resourcespb.JobResource + jobHandlers map[string]*batchpb.RegistrationRequest + + policies []*resourcespb.PolicyResource + + errors []error + topicspb.UnimplementedTopicsServer + storagepb.UnimplementedStorageListenerServer + websocketspb.UnimplementedWebsocketServer + + ApiServer apispb.ApiServer +} + +// Error - Returns an error if any requirements have been registered incorrectly, such as duplicates +func (s *BatchRequirements) Error() error { + if len(s.errors) > 0 { + errorView := view.New() + errorView.Addln("Errors found in batch %s", s.batchFile) + + for _, err := range s.errors { + errorView.Addln("- %s", err.Error()) + } + + return errors.New(errorView.Render()) + } + + return nil +} + +// TODO: Remove when databases are no longer in preview +func (s *BatchRequirements) HasDatabases() bool { + return len(s.sqlDatabases) > 0 +} + +func (s *BatchRequirements) RegisterServices(grpcServer *grpc.Server) { + batchpb.RegisterJobServer(grpcServer, s) + resourcespb.RegisterResourcesServer(grpcServer, s) + apispb.RegisterApiServer(grpcServer, s.ApiServer) + schedulespb.RegisterSchedulesServer(grpcServer, s) + topicspb.RegisterTopicsServer(grpcServer, s) + topicspb.RegisterSubscriberServer(grpcServer, s) + websocketspb.RegisterWebsocketHandlerServer(grpcServer, s) + storagepb.RegisterStorageListenerServer(grpcServer, s) + httppb.RegisterHttpServer(grpcServer, s) +} + +func (s *BatchRequirements) Declare(ctx context.Context, req *resourcespb.ResourceDeclareRequest) (*resourcespb.ResourceDeclareResponse, error) { + s.resourceLock.Lock() + defer s.resourceLock.Unlock() + + switch req.Id.Type { + case resourcespb.ResourceType_Bucket: + // Add a bucket + s.buckets[req.Id.GetName()] = req.GetBucket() + case resourcespb.ResourceType_KeyValueStore: + // Add a key/value store + s.keyValueStores[req.Id.GetName()] = req.GetKeyValueStore() + case resourcespb.ResourceType_Api: + // Discard and ignore for batches + case resourcespb.ResourceType_ApiSecurityDefinition: + // Discard and ignore for batches + case resourcespb.ResourceType_Secret: + // Add a secret + s.secrets[req.Id.GetName()] = req.GetSecret() + case resourcespb.ResourceType_SqlDatabase: + // Add a sql database + s.sqlDatabases[req.Id.GetName()] = req.GetSqlDatabase() + case resourcespb.ResourceType_Policy: + // Services don't know their own name, so we need to add it here + if req.GetPolicy().GetPrincipals() == nil { + req.GetPolicy().Principals = []*resourcespb.ResourceIdentifier{{ + Name: s.batchName, + Type: resourcespb.ResourceType_Batch, + }} + } else { + for _, principal := range req.GetPolicy().GetPrincipals() { + if principal.GetName() == "" && principal.GetType() == resourcespb.ResourceType_Service { + principal.Name = s.batchName + principal.Type = resourcespb.ResourceType_Batch + } + } + } + + // Add a policy + s.policies = append(s.policies, req.GetPolicy()) + case resourcespb.ResourceType_Topic: + // add a topic + s.topics[req.Id.GetName()] = req.GetTopic() + case resourcespb.ResourceType_Queue: + // add a queue + s.queues[req.Id.GetName()] = req.GetQueue() + case resourcespb.ResourceType_Job: + // add a job + s.jobs[req.Id.GetName()] = req.GetJob() + } + + return &resourcespb.ResourceDeclareResponse{}, nil +} + +func (s *BatchRequirements) HandleJob(stream batchpb.Job_HandleJobServer) error { + s.resourceLock.Lock() + defer s.resourceLock.Unlock() + + msg, err := stream.Recv() + if err != nil { + return err + } + + registrationRequest := msg.GetRegistrationRequest() + if registrationRequest == nil { + return fmt.Errorf("first message must be a registration request") + } + + s.jobHandlers[registrationRequest.JobName] = registrationRequest + + return stream.Send(&batchpb.ServerMessage{ + Content: &batchpb.ServerMessage_RegistrationResponse{ + RegistrationResponse: &batchpb.RegistrationResponse{}, + }, + }) +} + +func (s *BatchRequirements) HandleEvents(stream websocketspb.WebsocketHandler_HandleEventsServer) error { + s.resourceLock.Lock() + defer s.resourceLock.Unlock() + + _, err := stream.Recv() + if err != nil { + return err + } + + s.errors = append(s.errors, fmt.Errorf("websocket handler declared in Batch %s, batches cannot handle Websocket events", s.batchFile)) + + return stream.Send(&websocketspb.ServerMessage{ + Content: &websocketspb.ServerMessage_RegistrationResponse{ + RegistrationResponse: &websocketspb.RegistrationResponse{}, + }, + }) +} + +func (s *BatchRequirements) Proxy(stream httppb.Http_ProxyServer) error { + s.resourceLock.Lock() + defer s.resourceLock.Unlock() + + _, err := stream.Recv() + if err != nil { + return err + } + + s.errors = append(s.errors, fmt.Errorf("HTTP Proxy declared in Batch %s, batches cannot handle HTTP servers", s.batchFile)) + + return nil +} + +func (s *BatchRequirements) Serve(stream apispb.Api_ServeServer) error { + s.resourceLock.Lock() + defer s.resourceLock.Unlock() + + _, err := stream.Recv() + if err != nil { + return err + } + + s.errors = append(s.errors, fmt.Errorf("API route declared in Batch %s, batches cannot handle API requests", s.batchFile)) + + // Send a registration response + return stream.Send(&apispb.ServerMessage{ + Content: &apispb.ServerMessage_RegistrationResponse{ + RegistrationResponse: &apispb.RegistrationResponse{}, + }, + }) +} + +func (s *BatchRequirements) Schedule(stream schedulespb.Schedules_ScheduleServer) error { + s.resourceLock.Lock() + defer s.resourceLock.Unlock() + + _, err := stream.Recv() + if err != nil { + return err + } + + s.errors = append(s.errors, fmt.Errorf("Schedule declared in Batch %s, batches cannot currently handle schedules", s.batchFile)) + + return stream.Send(&schedulespb.ServerMessage{ + Content: &schedulespb.ServerMessage_RegistrationResponse{ + RegistrationResponse: &schedulespb.RegistrationResponse{}, + }, + }) +} + +func (s *BatchRequirements) Subscribe(stream topicspb.Subscriber_SubscribeServer) error { + s.resourceLock.Lock() + defer s.resourceLock.Unlock() + + _, err := stream.Recv() + if err != nil { + return err + } + + s.errors = append(s.errors, fmt.Errorf("topic subscription declared in Batch %s, batches cannot handle topic subscriptions", s.batchFile)) + + return stream.Send(&topicspb.ServerMessage{ + Content: &topicspb.ServerMessage_RegistrationResponse{ + RegistrationResponse: &topicspb.RegistrationResponse{}, + }, + }) +} + +func NewBatchRequirements(serviceName string, serviceFile string) *BatchRequirements { + requirements := &BatchRequirements{ + batchName: serviceName, + batchFile: serviceFile, + resourceLock: sync.Mutex{}, + jobHandlers: make(map[string]*batchpb.RegistrationRequest), + jobs: make(map[string]*resourcespb.JobResource), + buckets: make(map[string]*resourcespb.BucketResource), + keyValueStores: make(map[string]*resourcespb.KeyValueStoreResource), + topics: make(map[string]*resourcespb.TopicResource), + policies: []*resourcespb.PolicyResource{}, + secrets: make(map[string]*resourcespb.SecretResource), + sqlDatabases: make(map[string]*resourcespb.SqlDatabaseResource), + queues: make(map[string]*resourcespb.QueueResource), + errors: []error{}, + } + + return requirements +} diff --git a/pkg/collector/service.go b/pkg/collector/service.go index a97a93569..caf50f8dd 100644 --- a/pkg/collector/service.go +++ b/pkg/collector/service.go @@ -23,6 +23,7 @@ import ( "sync" "github.com/samber/lo" + "google.golang.org/grpc" "github.com/nitrictech/cli/pkg/view/tui/components/view" apispb "github.com/nitrictech/nitric/core/pkg/proto/apis/v1" @@ -58,11 +59,15 @@ type ServiceRequirements struct { topics map[string]*resourcespb.TopicResource queues map[string]*resourcespb.QueueResource sqlDatabases map[string]*resourcespb.SqlDatabaseResource + jobs map[string]*resourcespb.JobResource policies []*resourcespb.PolicyResource secrets map[string]*resourcespb.SecretResource errors []error + topicspb.UnimplementedTopicsServer + storagepb.UnimplementedStorageListenerServer + websocketspb.UnimplementedWebsocketServer ApiServer apispb.ApiServer } @@ -164,6 +169,9 @@ func (s *ServiceRequirements) Declare(ctx context.Context, req *resourcespb.Reso case resourcespb.ResourceType_Queue: // add a queue s.queues[req.Id.GetName()] = req.GetQueue() + case resourcespb.ResourceType_Job: + // add a job + s.jobs[req.Id.GetName()] = req.GetJob() } return &resourcespb.ResourceDeclareResponse{}, nil @@ -315,6 +323,17 @@ func (s *ServiceRequirements) Listen(stream storagepb.StorageListener_ListenServ }) } +func (s *ServiceRequirements) RegisterServices(grpcServer *grpc.Server) { + resourcespb.RegisterResourcesServer(grpcServer, s) + apispb.RegisterApiServer(grpcServer, s.ApiServer) + schedulespb.RegisterSchedulesServer(grpcServer, s) + topicspb.RegisterTopicsServer(grpcServer, s) + topicspb.RegisterSubscriberServer(grpcServer, s) + websocketspb.RegisterWebsocketHandlerServer(grpcServer, s) + storagepb.RegisterStorageListenerServer(grpcServer, s) + httppb.RegisterHttpServer(grpcServer, s) +} + func (s *ServiceRequirements) HandleEvents(stream websocketspb.WebsocketHandler_HandleEventsServer) error { s.resourceLock.Lock() defer s.resourceLock.Unlock() @@ -371,6 +390,7 @@ func NewServiceRequirements(serviceName string, serviceFile string, serviceType sqlDatabases: make(map[string]*resourcespb.SqlDatabaseResource), apiSecurityDefinition: make(map[string]map[string]*resourcespb.ApiSecurityDefinitionResource), queues: make(map[string]*resourcespb.QueueResource), + jobs: make(map[string]*resourcespb.JobResource), errors: []error{}, } requirements.ApiServer = &ApiCollectorServer{ diff --git a/pkg/collector/spec.go b/pkg/collector/spec.go index eee9cb651..5bbc5a88d 100644 --- a/pkg/collector/spec.go +++ b/pkg/collector/spec.go @@ -35,6 +35,7 @@ import ( "github.com/nitrictech/cli/pkg/project/runtime" "github.com/nitrictech/cli/pkg/view/tui/components/view" apispb "github.com/nitrictech/nitric/core/pkg/proto/apis/v1" + batchpb "github.com/nitrictech/nitric/core/pkg/proto/batch/v1" deploymentspb "github.com/nitrictech/nitric/core/pkg/proto/deployments/v1" resourcespb "github.com/nitrictech/nitric/core/pkg/proto/resources/v1" schedulespb "github.com/nitrictech/nitric/core/pkg/proto/schedules/v1" @@ -66,7 +67,7 @@ func (pe ProjectErrors) Error() error { } // buildBucketRequirements gathers and deduplicates all bucket requirements -func buildBucketRequirements(allServiceRequirements []*ServiceRequirements, projectErrors *ProjectErrors) ([]*deploymentspb.Resource, error) { +func buildBucketRequirements(allServiceRequirements []*ServiceRequirements, allBatchRequirements []*BatchRequirements, projectErrors *ProjectErrors) ([]*deploymentspb.Resource, error) { resources := []*deploymentspb.Resource{} for _, serviceRequirements := range allServiceRequirements { @@ -106,6 +107,32 @@ func buildBucketRequirements(allServiceRequirements []*ServiceRequirements, proj } } + // TODO: Consolidate duplicate code for batch requirement handling + for _, batchRequirements := range allBatchRequirements { + for bucketName := range batchRequirements.buckets { + notifications := []*deploymentspb.BucketListener{} + + _, exists := lo.Find(resources, func(item *deploymentspb.Resource) bool { + return item.Id.Name == bucketName + }) + + if !exists { + res := &deploymentspb.Resource{ + Id: &resourcespb.ResourceIdentifier{ + Name: bucketName, + Type: resourcespb.ResourceType_Bucket, + }, + Config: &deploymentspb.Resource_Bucket{ + Bucket: &deploymentspb.Bucket{ + Listeners: notifications, + }, + }, + } + resources = append(resources, res) + } + } + } + return resources, nil } @@ -176,54 +203,66 @@ func MakeDatabaseServiceRequirements(sqlDatabases map[string]*resourcespb.SqlDat // Collect a list of migration images that need to be built // these requirements need to be supplied to the deployment serviceS -func GetMigrationImageBuildContexts(allServiceRequirements []*ServiceRequirements, fs afero.Fs) (map[string]*runtime.RuntimeBuildContext, error) { +func GetMigrationImageBuildContexts(allServiceRequirements []*ServiceRequirements, allBatchRequirements []*BatchRequirements, fs afero.Fs) (map[string]*runtime.RuntimeBuildContext, error) { imageBuildContexts := map[string]*runtime.RuntimeBuildContext{} declaredConfigs := map[string]string{} + sqlDbs := map[string]*resourcespb.SqlDatabaseResource{} + for _, serviceRequirements := range allServiceRequirements { for databaseName, databaseConfig := range serviceRequirements.sqlDatabases { - if databaseConfig.Migrations != nil && databaseConfig.Migrations.GetMigrationsPath() != "" { - scheme, path, err := parseMigrationsScheme(databaseConfig.Migrations.GetMigrationsPath()) - if err != nil { - return nil, err - } + sqlDbs[databaseName] = databaseConfig + } + } - // if the db has already been declared check that it dies not differ from a previous declaration - if _, exists := imageBuildContexts[databaseName]; exists { - if declaredConfigs[databaseName] != databaseConfig.Migrations.GetMigrationsPath() { - return nil, fmt.Errorf("multiple migrations paths declared for database '%s'", databaseName) - } - // otherwise set named config to the already build config - continue + for _, batchRequirements := range allBatchRequirements { + for databaseName, databaseConfig := range batchRequirements.sqlDatabases { + sqlDbs[databaseName] = databaseConfig + } + } + + for databaseName, databaseConfig := range sqlDbs { + if databaseConfig.Migrations != nil && databaseConfig.Migrations.GetMigrationsPath() != "" { + scheme, path, err := parseMigrationsScheme(databaseConfig.Migrations.GetMigrationsPath()) + if err != nil { + return nil, err + } + + // if the db has already been declared check that it dies not differ from a previous declaration + if _, exists := imageBuildContexts[databaseName]; exists { + if declaredConfigs[databaseName] != databaseConfig.Migrations.GetMigrationsPath() { + return nil, fmt.Errorf("multiple migrations paths declared for database '%s'", databaseName) } + // otherwise set named config to the already build config + continue + } - declaredConfigs[databaseName] = databaseConfig.Migrations.GetMigrationsPath() + declaredConfigs[databaseName] = databaseConfig.Migrations.GetMigrationsPath() - switch scheme { - case "dockerfile": - // Read the referenced dockerfile - dockerfileContents, err := afero.ReadFile(fs, path) - if err != nil { - return nil, err - } + switch scheme { + case "dockerfile": + // Read the referenced dockerfile + dockerfileContents, err := afero.ReadFile(fs, path) + if err != nil { + return nil, err + } - imageBuildContexts[databaseName] = &runtime.RuntimeBuildContext{ - BuildArguments: map[string]string{}, - DockerfileContents: string(dockerfileContents), - BaseDirectory: ".", - } - case "file": - // Default dockerfile build context for the given path - imageBuildContexts[databaseName] = &runtime.RuntimeBuildContext{ - BuildArguments: map[string]string{ - "MIGRATIONS_PATH": path, - }, - DockerfileContents: defaultMigrationFileContents, - BaseDirectory: ".", - } - default: - return nil, fmt.Errorf("unsupported migration path scheme: %s, must be one of dockerfile or file", scheme) + imageBuildContexts[databaseName] = &runtime.RuntimeBuildContext{ + BuildArguments: map[string]string{}, + DockerfileContents: string(dockerfileContents), + BaseDirectory: ".", } + case "file": + // Default dockerfile build context for the given path + imageBuildContexts[databaseName] = &runtime.RuntimeBuildContext{ + BuildArguments: map[string]string{ + "MIGRATIONS_PATH": path, + }, + DockerfileContents: defaultMigrationFileContents, + BaseDirectory: ".", + } + default: + return nil, fmt.Errorf("unsupported migration path scheme: %s, must be one of dockerfile or file", scheme) } } } @@ -231,11 +270,21 @@ func GetMigrationImageBuildContexts(allServiceRequirements []*ServiceRequirement return imageBuildContexts, nil } -func buildDatabaseRequirements(allServiceRequirements []*ServiceRequirements, projectErrors *ProjectErrors) ([]*deploymentspb.Resource, error) { +func buildDatabaseRequirements(allServiceRequirements []*ServiceRequirements, allBatchRequirements []*BatchRequirements, projectErrors *ProjectErrors) ([]*deploymentspb.Resource, error) { resources := []*deploymentspb.Resource{} + allDatabases := []map[string]*resourcespb.SqlDatabaseResource{} + for _, serviceRequirements := range allServiceRequirements { - for databaseName, dbConfig := range serviceRequirements.sqlDatabases { + allDatabases = append(allDatabases, serviceRequirements.sqlDatabases) + } + + for _, batchRequirements := range allBatchRequirements { + allDatabases = append(allDatabases, batchRequirements.sqlDatabases) + } + + for _, dbs := range allDatabases { + for databaseName, dbConfig := range dbs { _, exists := lo.Find(resources, func(item *deploymentspb.Resource) bool { return item.Id.Name == databaseName }) @@ -269,7 +318,7 @@ func buildDatabaseRequirements(allServiceRequirements []*ServiceRequirements, pr } // buildTopicRequirements gathers and deduplicates all topic requirements -func buildTopicRequirements(allServiceRequirements []*ServiceRequirements, projectErrors *ProjectErrors) ([]*deploymentspb.Resource, error) { +func buildTopicRequirements(allServiceRequirements []*ServiceRequirements, allBatchRequirements []*BatchRequirements, projectErrors *ProjectErrors) ([]*deploymentspb.Resource, error) { resources := []*deploymentspb.Resource{} for _, serviceRequirements := range allServiceRequirements { @@ -303,15 +352,50 @@ func buildTopicRequirements(allServiceRequirements []*ServiceRequirements, proje } } + // FIXME: Reduce duplicate code + // TODO: May be unnecessary as any topic requirements here would be publishing for services to respond to already + for _, batchRequirements := range allBatchRequirements { + for topicName := range batchRequirements.topics { + _, exists := lo.Find(resources, func(item *deploymentspb.Resource) bool { + return item.Id.Name == topicName + }) + + if !exists { + res := &deploymentspb.Resource{ + Id: &resourcespb.ResourceIdentifier{ + Name: topicName, + Type: resourcespb.ResourceType_Topic, + }, + Config: &deploymentspb.Resource_Topic{ + Topic: &deploymentspb.Topic{ + Subscriptions: []*deploymentspb.SubscriptionTarget{}, + }, + }, + } + resources = append(resources, res) + } + } + } + return resources, nil } // buildQueueRequirements gathers and deduplicates all queue requirements -func buildQueueRequirements(allServiceRequirements []*ServiceRequirements, projectErrors *ProjectErrors) ([]*deploymentspb.Resource, error) { +func buildQueueRequirements(allServiceRequirements []*ServiceRequirements, allBatchRequirements []*BatchRequirements, projectErrors *ProjectErrors) ([]*deploymentspb.Resource, error) { resources := []*deploymentspb.Resource{} + allQueues := []map[string]*resourcespb.QueueResource{} + for _, serviceRequirements := range allServiceRequirements { - for queueName := range serviceRequirements.queues { + allQueues = append(allQueues, serviceRequirements.queues) + } + + for _, batchRequirements := range allBatchRequirements { + allQueues = append(allQueues, batchRequirements.queues) + } + + for _, queues := range allQueues { + for queueName := range queues { _, exists := lo.Find(resources, func(item *deploymentspb.Resource) bool { return item.Id.Name == queueName }) @@ -335,11 +419,21 @@ func buildQueueRequirements(allServiceRequirements []*ServiceRequirements, proje } // buildSecretRequirements gathers and deduplicates all secret requirements -func buildSecretRequirements(allServiceRequirements []*ServiceRequirements, projectErrors *ProjectErrors) ([]*deploymentspb.Resource, error) { +func buildSecretRequirements(allServiceRequirements []*ServiceRequirements, allBatchRequirements []*BatchRequirements, projectErrors *ProjectErrors) ([]*deploymentspb.Resource, error) { resources := []*deploymentspb.Resource{} + allSecrets := []map[string]*resourcespb.SecretResource{} + for _, serviceRequirements := range allServiceRequirements { - for secretName := range serviceRequirements.secrets { + allSecrets = append(allSecrets, serviceRequirements.secrets) + } + + for _, batchRequirements := range allBatchRequirements { + allSecrets = append(allSecrets, batchRequirements.secrets) + } + + for _, secrets := range allSecrets { + for secretName := range secrets { _, exists := lo.Find(resources, func(item *deploymentspb.Resource) bool { return item.Id.Name == secretName }) @@ -722,12 +816,53 @@ func policyResourceName(policy *resourcespb.PolicyResource) (string, error) { return hex.EncodeToString(hasher.Sum(nil)), nil } +func checkJobHandlers(allServiceRequirements []*ServiceRequirements, allBatchRequirements []*BatchRequirements, projectErrors *ProjectErrors) { + allJobs := map[string]*resourcespb.JobResource{} + allJobHandlers := map[string]*batchpb.RegistrationRequest{} + + for _, serviceRequirements := range allServiceRequirements { + for jobName, jobConfig := range serviceRequirements.jobs { + allJobs[jobName] = jobConfig + } + } + + for _, batchRequirements := range allBatchRequirements { + for jobName, jobConfig := range batchRequirements.jobs { + allJobs[jobName] = jobConfig + } + + for jobHandlerName, jobHandlerConfig := range batchRequirements.jobHandlers { + if _, exists := allJobHandlers[jobHandlerName]; exists { + projectErrors.Add(fmt.Errorf("multiple handlers registered for job %s', jobs may only have one handler", jobHandlerName)) + } + + allJobHandlers[jobHandlerName] = jobHandlerConfig + } + } + + for jobName := range allJobs { + if _, exists := allJobHandlers[jobName]; !exists { + projectErrors.Add(fmt.Errorf("no handler registered for job '%s'", jobName)) + } + } +} + // buildKeyValueRequirements gathers and deduplicates all key/value requirements -func buildKeyValueRequirements(allServiceRequirements []*ServiceRequirements, projectErrors *ProjectErrors) ([]*deploymentspb.Resource, error) { +func buildKeyValueRequirements(allServiceRequirements []*ServiceRequirements, allBatchRequirements []*BatchRequirements, projectErrors *ProjectErrors) ([]*deploymentspb.Resource, error) { resources := []*deploymentspb.Resource{} + allKeyValueStores := []map[string]*resourcespb.KeyValueStoreResource{} + for _, serviceRequirements := range allServiceRequirements { - for kvStoreName := range serviceRequirements.keyValueStores { + allKeyValueStores = append(allKeyValueStores, serviceRequirements.keyValueStores) + } + + for _, batchRequirements := range allBatchRequirements { + allKeyValueStores = append(allKeyValueStores, batchRequirements.keyValueStores) + } + + for _, kvStores := range allKeyValueStores { + for kvStoreName := range kvStores { _, exists := lo.Find(resources, func(item *deploymentspb.Resource) bool { return item.Id.Name == kvStoreName }) @@ -750,11 +885,21 @@ func buildKeyValueRequirements(allServiceRequirements []*ServiceRequirements, pr // buildPolicyRequirements gathers, compacts, and deduplicates all policy requirements // compaction is done by grouping policies by their principals and actions // i.e. two or more policies with identical principals and actions, but different resources, will be combined into a single policy covering all resources. -func buildPolicyRequirements(allServiceRequirements []*ServiceRequirements, projectErrors *ProjectErrors) ([]*deploymentspb.Resource, error) { +func buildPolicyRequirements(allServiceRequirements []*ServiceRequirements, allBatchRequirements []*BatchRequirements, projectErrors *ProjectErrors) ([]*deploymentspb.Resource, error) { resources := []*deploymentspb.Resource{} + allPolicies := [][]*resourcespb.PolicyResource{} + for _, serviceRequirements := range allServiceRequirements { - compactedPoliciesByKey := lo.GroupBy(serviceRequirements.policies, func(item *resourcespb.PolicyResource) string { + allPolicies = append(allPolicies, serviceRequirements.policies) + } + + for _, batchRequirements := range allBatchRequirements { + allPolicies = append(allPolicies, batchRequirements.policies) + } + + for _, policies := range allPolicies { + compactedPoliciesByKey := lo.GroupBy(policies, func(item *resourcespb.PolicyResource) string { // get the princpals and actions as a unique key (make sure they're sorted for consistency) principalNames := lo.Reduce(item.Principals, func(agg []string, principal *resourcespb.ResourceIdentifier, idx int) []string { return append(agg, principal.Name) @@ -855,7 +1000,7 @@ func checkServiceRequirementErrors(allServiceRequirements []*ServiceRequirements } // convert service requirements to a cloud bill of materials -func ServiceRequirementsToSpec(projectName string, environmentVariables map[string]string, allServiceRequirements []*ServiceRequirements, defaultMigrationImage string) (*deploymentspb.Spec, error) { +func ServiceRequirementsToSpec(projectName string, environmentVariables map[string]string, allServiceRequirements []*ServiceRequirements, allBatchRequirements []*BatchRequirements) (*deploymentspb.Spec, error) { if err := checkServiceRequirementErrors(allServiceRequirements); err != nil { return nil, err } @@ -866,35 +1011,38 @@ func ServiceRequirementsToSpec(projectName string, environmentVariables map[stri Resources: []*deploymentspb.Resource{}, } - databaseResources, err := buildDatabaseRequirements(allServiceRequirements, projectErrors) + // Check for duplicate/missing job handlers and update projectErrors with misconfigration + checkJobHandlers(allServiceRequirements, allBatchRequirements, projectErrors) + + databaseResources, err := buildDatabaseRequirements(allServiceRequirements, allBatchRequirements, projectErrors) if err != nil { return nil, err } newSpec.Resources = append(newSpec.Resources, databaseResources...) - bucketResources, err := buildBucketRequirements(allServiceRequirements, projectErrors) + bucketResources, err := buildBucketRequirements(allServiceRequirements, allBatchRequirements, projectErrors) if err != nil { return nil, err } newSpec.Resources = append(newSpec.Resources, bucketResources...) - topicResources, err := buildTopicRequirements(allServiceRequirements, projectErrors) + topicResources, err := buildTopicRequirements(allServiceRequirements, allBatchRequirements, projectErrors) if err != nil { return nil, err } newSpec.Resources = append(newSpec.Resources, topicResources...) - queueResources, err := buildQueueRequirements(allServiceRequirements, projectErrors) + queueResources, err := buildQueueRequirements(allServiceRequirements, allBatchRequirements, projectErrors) if err != nil { return nil, err } newSpec.Resources = append(newSpec.Resources, queueResources...) - secretResrources, err := buildSecretRequirements(allServiceRequirements, projectErrors) + secretResrources, err := buildSecretRequirements(allServiceRequirements, allBatchRequirements, projectErrors) if err != nil { return nil, err } @@ -929,14 +1077,14 @@ func ServiceRequirementsToSpec(projectName string, environmentVariables map[stri newSpec.Resources = append(newSpec.Resources, apiResources...) - keyValueResources, err := buildKeyValueRequirements(allServiceRequirements, projectErrors) + keyValueResources, err := buildKeyValueRequirements(allServiceRequirements, allBatchRequirements, projectErrors) if err != nil { return nil, err } newSpec.Resources = append(newSpec.Resources, keyValueResources...) - policyResources, err := buildPolicyRequirements(allServiceRequirements, projectErrors) + policyResources, err := buildPolicyRequirements(allServiceRequirements, allBatchRequirements, projectErrors) if err != nil { return nil, err } @@ -964,6 +1112,32 @@ func ServiceRequirementsToSpec(projectName string, environmentVariables map[stri }) } + for _, batchRequirements := range allBatchRequirements { + newSpec.Resources = append(newSpec.Resources, &deploymentspb.Resource{ + Id: &resourcespb.ResourceIdentifier{ + Name: batchRequirements.batchName, + Type: resourcespb.ResourceType_Batch, + }, + Config: &deploymentspb.Resource_Batch{ + Batch: &deploymentspb.Batch{ + Source: &deploymentspb.Batch_Image{ + Image: &deploymentspb.ImageSource{ + Uri: fmt.Sprintf(batchRequirements.batchName), + }, + }, + Type: "default", + Env: environmentVariables, + Jobs: lo.Map(lo.Entries(batchRequirements.jobHandlers), func(item lo.Entry[string, *batchpb.RegistrationRequest], idx int) *deploymentspb.Job { + return &deploymentspb.Job{ + Name: item.Key, + Requirements: item.Value.Requirements, + } + }), + }, + }, + }) + } + return newSpec, projectErrors.Error() } diff --git a/pkg/dashboard/dashboard.go b/pkg/dashboard/dashboard.go index 3bf5b2bba..a44095345 100644 --- a/pkg/dashboard/dashboard.go +++ b/pkg/dashboard/dashboard.go @@ -17,6 +17,7 @@ package dashboard import ( + "context" "embed" "encoding/json" "fmt" @@ -40,6 +41,7 @@ import ( "github.com/nitrictech/cli/pkg/collector" "github.com/nitrictech/cli/pkg/netx" resourcespb "github.com/nitrictech/nitric/core/pkg/proto/resources/v1" + sqlpb "github.com/nitrictech/nitric/core/pkg/proto/sql/v1" websocketspb "github.com/nitrictech/nitric/core/pkg/proto/websockets/v1" "github.com/nitrictech/cli/pkg/cloud/apis" @@ -119,6 +121,12 @@ type SubscriberSpec struct { Target string `json:"target"` } +type BatchJobSpec struct { + *BaseResourceSpec + + Target string `json:"target"` +} + type PolicyResource struct { Name string `json:"name"` Type string `json:"type"` @@ -137,6 +145,12 @@ type ServiceSpec struct { FilePath string `json:"filePath"` } +type BatchSpec struct { + *BaseResourceSpec + + FilePath string `json:"filePath"` +} + type HttpProxySpec struct { *BaseResourceSpec @@ -155,6 +169,7 @@ type Dashboard struct { apiSecurityDefinitions map[string]map[string]*resourcespb.ApiSecurityDefinitionResource schedules []ScheduleSpec topics []*TopicSpec + batchJobs []*BatchJobSpec buckets []*BucketSpec stores []*KeyValueSpec secrets []*SecretSpec @@ -181,6 +196,8 @@ type Dashboard struct { type DashboardResponse struct { Apis []ApiSpec `json:"apis"` ApisUseHttps bool `json:"apisUseHttps"` + Batches []*BatchSpec `json:"batchServices"` + BatchJobs []*BatchJobSpec `json:"jobs"` Buckets []*BucketSpec `json:"buckets"` Schedules []ScheduleSpec `json:"schedules"` Topics []*TopicSpec `json:"topics"` @@ -235,11 +252,32 @@ func (d *Dashboard) getServices() ([]*ServiceSpec, error) { return serviceSpecs, nil } +func (d *Dashboard) getBatchServices() ([]*BatchSpec, error) { + batchSpecs := []*BatchSpec{} + + for _, batch := range d.project.GetBatchServices() { + absPath, err := batch.GetAbsoluteFilePath() + if err != nil { + return nil, err + } + + batchSpecs = append(batchSpecs, &BatchSpec{ + BaseResourceSpec: &BaseResourceSpec{ + Name: batch.GetFilePath(), + }, + FilePath: absPath, + }) + } + + return batchSpecs, nil +} + func (d *Dashboard) updateResources(lrs resources.LocalResourcesState) { d.resourcesLock.Lock() defer d.resourcesLock.Unlock() d.buckets = []*BucketSpec{} + d.batchJobs = []*BatchJobSpec{} d.topics = []*TopicSpec{} d.stores = []*KeyValueSpec{} d.queues = []*QueueSpec{} @@ -340,6 +378,57 @@ func (d *Dashboard) updateResources(lrs resources.LocalResourcesState) { }) } + for dbName, resource := range lrs.SqlDatabases.GetAll() { + exists := lo.ContainsBy(d.sqlDatabases, func(item *SQLDatabaseSpec) bool { + return item.Name == dbName + }) + + if !exists { + connectionString, err := d.databaseService.ConnectionString(context.TODO(), &sqlpb.SqlConnectionStringRequest{ + DatabaseName: dbName, + }) + if err != nil { + fmt.Printf("Error getting connection string for database %s: %v\n", dbName, err) + continue + } + + d.sqlDatabases = append(d.sqlDatabases, &SQLDatabaseSpec{ + BaseResourceSpec: &BaseResourceSpec{ + Name: dbName, + RequestingServices: resource.RequestingServices, + }, + ConnectionString: connectionString.GetConnectionString(), + }) + } + } + + if len(d.sqlDatabases) > 0 { + slices.SortFunc(d.sqlDatabases, func(a, b *SQLDatabaseSpec) int { + return compare(a.Name, b.Name) + }) + } + + for jobName, resource := range lrs.BatchJobs.GetAll() { + exists := lo.ContainsBy(d.batchJobs, func(item *BatchJobSpec) bool { + return item.Name == jobName + }) + + if !exists { + d.batchJobs = append(d.batchJobs, &BatchJobSpec{ + BaseResourceSpec: &BaseResourceSpec{ + Name: jobName, + RequestingServices: resource.RequestingServices, + }, + }) + } + } + + if len(d.batchJobs) > 0 { + slices.SortFunc(d.batchJobs, func(a, b *BatchJobSpec) int { + return compare(a.Name, b.Name) + }) + } + for policyName, policy := range lrs.Policies.GetAll() { d.policies[policyName] = PolicySpec{ BaseResourceSpec: &BaseResourceSpec{ @@ -577,6 +666,7 @@ func (d *Dashboard) refresh() { func (d *Dashboard) isConnected() bool { apisRegistered := len(d.apis) > 0 + batchServicesRegistered := len(d.batchJobs) > 0 websocketsRegistered := len(d.websockets) > 0 topicsRegistered := len(d.topics) > 0 schedulesRegistered := len(d.schedules) > 0 @@ -586,7 +676,7 @@ func (d *Dashboard) isConnected() bool { sqlRegistered := len(d.sqlDatabases) > 0 secretsRegistered := len(d.secrets) > 0 - return apisRegistered || websocketsRegistered || topicsRegistered || schedulesRegistered || notificationsRegistered || proxiesRegistered || storesRegistered || sqlRegistered || secretsRegistered + return apisRegistered || batchServicesRegistered || websocketsRegistered || topicsRegistered || schedulesRegistered || notificationsRegistered || proxiesRegistered || storesRegistered || sqlRegistered || secretsRegistered } func (d *Dashboard) Start() error { @@ -745,9 +835,16 @@ func (d *Dashboard) sendStackUpdate() error { return err } + batchServices, err := d.getBatchServices() + if err != nil { + return err + } + response := &DashboardResponse{ Apis: d.apis, Topics: d.topics, + Batches: batchServices, + BatchJobs: d.batchJobs, Buckets: d.buckets, Stores: d.stores, SQLDatabases: d.sqlDatabases, @@ -833,6 +930,7 @@ func New(noBrowser bool, localCloud *cloud.LocalCloud, project *project.Project) stackWebSocket: stackWebSocket, historyWebSocket: historyWebSocket, wsWebSocket: wsWebSocket, + batchJobs: []*BatchJobSpec{}, buckets: []*BucketSpec{}, schedules: []ScheduleSpec{}, topics: []*TopicSpec{}, @@ -874,6 +972,7 @@ func New(noBrowser bool, localCloud *cloud.LocalCloud, project *project.Project) localCloud.Apis.SubscribeToAction(dash.handleApiHistory) localCloud.Topics.SubscribeToAction(dash.handleTopicsHistory) localCloud.Schedules.SubscribeToAction(dash.handleSchedulesHistory) + localCloud.Batch.SubscribeToAction(dash.handleBatchJobsHistory) localCloud.Websockets.SubscribeToAction(dash.handleWebsocketEvents) return dash, nil diff --git a/pkg/dashboard/frontend/src/components/architecture/nodes/BatchNode.tsx b/pkg/dashboard/frontend/src/components/architecture/nodes/BatchNode.tsx new file mode 100644 index 000000000..80bb297cc --- /dev/null +++ b/pkg/dashboard/frontend/src/components/architecture/nodes/BatchNode.tsx @@ -0,0 +1,39 @@ +import { type ComponentType } from 'react' + +import type { Edge, NodeProps } from 'reactflow' +import NodeBase, { type NodeBaseData } from './NodeBase' +import { Button } from '@/components/ui/button' + +type BatchData = { + filePath: string +} + +export interface BatchNodeData extends NodeBaseData { + connectedEdges: Edge[] +} + +export const BatchNode: ComponentType> = (props) => { + const { data } = props + + const Icon = data.icon + + return ( + + + + Open in VSCode + + + ), + }} + /> + ) +} diff --git a/pkg/dashboard/frontend/src/components/architecture/nodes/JobNode.tsx b/pkg/dashboard/frontend/src/components/architecture/nodes/JobNode.tsx new file mode 100644 index 000000000..43dee421a --- /dev/null +++ b/pkg/dashboard/frontend/src/components/architecture/nodes/JobNode.tsx @@ -0,0 +1,26 @@ +import { type ComponentType } from 'react' + +import type { BatchJob } from '@/types' +import type { NodeProps } from 'reactflow' +import NodeBase, { type NodeBaseData } from './NodeBase' + +export type JobNodeData = NodeBaseData + +export const JobNode: ComponentType> = (props) => { + const { data } = props + + return ( + + ) +} diff --git a/pkg/dashboard/frontend/src/components/architecture/styles.css b/pkg/dashboard/frontend/src/components/architecture/styles.css index 127f48133..e77bb7a6d 100644 --- a/pkg/dashboard/frontend/src/components/architecture/styles.css +++ b/pkg/dashboard/frontend/src/components/architecture/styles.css @@ -69,6 +69,13 @@ --nitric-node-icon-color: #059669; /* emerald 600 */ } +.react-flow__node-job { + --nitric-node-from: #ca8a04; /* yellow 600 */ + --nitric-node-via: #facc15; /* yellow 400 */ + --nitric-node-to: #a16207; /* yellow 700 */ + --nitric-node-icon-color: #ca8a04; /* yellow 600 */ +} + .react-flow__node-topic { --nitric-node-from: #d97706; /* amber 600 */ --nitric-node-via: #fbbf24; /* amber 400 */ @@ -97,6 +104,13 @@ --nitric-node-icon-color: #0284c7; /* Sky 600 */ } +.react-flow__node-batch { + --nitric-node-from: #ea580c; /* orange 600 */ + --nitric-node-via: #fb923c; /* orange 400 */ + --nitric-node-to: #c2410c; /* orange 700 */ + --nitric-node-icon-color: #ea580c; /* orange 600 */ +} + .react-flow__node-websocket { --nitric-node-from: #c026d3; /* Fuchsia 600 */ --nitric-node-via: #e879f9; /* Fuchsia 400 */ diff --git a/pkg/dashboard/frontend/src/components/events/EventsExplorer.tsx b/pkg/dashboard/frontend/src/components/events/EventsExplorer.tsx index 87c72889a..36aa5ac25 100644 --- a/pkg/dashboard/frontend/src/components/events/EventsExplorer.tsx +++ b/pkg/dashboard/frontend/src/components/events/EventsExplorer.tsx @@ -1,6 +1,12 @@ import { useEffect, useState } from 'react' import { useWebSocket } from '../../lib/hooks/use-web-socket' -import type { APIResponse, EventHistoryItem, Schedule, Topic } from '@/types' +import type { + APIResponse, + BatchJob, + EventHistoryItem, + Schedule, + Topic, +} from '@/types' import { Badge, Spinner, Tabs, Loading } from '../shared' import APIResponseContent from '../apis/APIResponseContent' import { @@ -20,7 +26,7 @@ import EventsTreeView from './EventsTreeView' import { copyToClipboard } from '../../lib/utils/copy-to-clipboard' import ClipboardIcon from '@heroicons/react/24/outline/ClipboardIcon' import toast from 'react-hot-toast' -import { capitalize } from 'radash' +import { title } from 'radash' import { Button } from '../ui/button' import { Tooltip, TooltipContent, TooltipTrigger } from '../ui/tooltip' import BreadCrumbs from '../layout/BreadCrumbs' @@ -35,13 +41,13 @@ import { import SectionCard from '../shared/SectionCard' interface Props { - workerType: 'schedules' | 'topics' + workerType: 'schedules' | 'topics' | 'jobs' } -type Worker = Schedule | Topic +type Worker = Schedule | Topic | BatchJob const EventsExplorer: React.FC = ({ workerType }) => { - const storageKey = `nitric-local-dash-${workerType}-history` + const storageKey = `nitric-local-dash-${workerType.toLowerCase()}-history` const { data, loading } = useWebSocket() const [callLoading, setCallLoading] = useState(false) @@ -59,7 +65,7 @@ const EventsExplorer: React.FC = ({ workerType }) => { useEffect(() => { if (history) { - setEventHistory(history ? history[workerType] : []) + setEventHistory(history ? history[workerType] ?? [] : []) } }, [history]) @@ -136,14 +142,14 @@ const EventsExplorer: React.FC = ({ workerType }) => { setTimeout(() => setCallLoading(false), 300) } - const workerTitleSingle = capitalize(workerType).slice(0, -1) + const workerTitleSingle = title(workerType).slice(0, -1) const generatedURL = `http://${data?.triggerAddress}/${workerType}/${selectedWorker?.name}` const hasData = Boolean(data && data[workerType]?.length) return ( = ({ workerType }) => { selectedWorker && ( <>
- {capitalize(workerType)} + {title(workerType)} = ({ workerType }) => { > @@ -217,7 +223,7 @@ const EventsExplorer: React.FC = ({ workerType }) => {
- {capitalize(workerType)} + {title(workerType)}

{selectedWorker.name}

@@ -270,7 +276,7 @@ const EventsExplorer: React.FC = ({ workerType }) => { )}
- {workerType === 'topics' && ( + {['jobs', 'topics'].includes(workerType) && (
= ({ workerType }) => { data-testid={`trigger-${workerType}-btn`} onClick={handleSend} > - {workerType === 'topics' ? 'Publish' : 'Trigger'} + { + { + schedules: 'Trigger', + topics: 'Publish', + jobs: 'Submit', + }[workerType] + }
@@ -421,7 +433,7 @@ const EventsExplorer: React.FC = ({ workerType }) => { href="https://nitric.io/docs/messaging" rel="noreferrer" > - creating {capitalize(workerType)} + creating {title(workerType)} {' '} as we are unable to find any existing {workerType}. diff --git a/pkg/dashboard/frontend/src/components/events/EventsHistory.tsx b/pkg/dashboard/frontend/src/components/events/EventsHistory.tsx index dc944bfda..157c9af44 100644 --- a/pkg/dashboard/frontend/src/components/events/EventsHistory.tsx +++ b/pkg/dashboard/frontend/src/components/events/EventsHistory.tsx @@ -1,7 +1,6 @@ import type { EventHistoryItem, - Schedule, - Topic, + EventResource, TopicHistoryItem, } from '../../types' import { formatJSON } from '@/lib/utils' @@ -10,8 +9,8 @@ import HistoryAccordion from '../shared/HistoryAccordion' interface Props { history: EventHistoryItem[] - selectedWorker: Schedule | Topic - workerType: 'schedules' | 'topics' + selectedWorker: EventResource + workerType: 'schedules' | 'topics' | 'jobs' } const EventsHistory: React.FC = ({ @@ -34,7 +33,7 @@ const EventsHistory: React.FC = ({ items={requestHistory.map((h) => { let payload = '' - if (workerType === 'topics') { + if (workerType === 'topics' || workerType === 'jobs') { payload = (h.event as TopicHistoryItem['event']).payload } diff --git a/pkg/dashboard/frontend/src/components/events/EventsTreeView.tsx b/pkg/dashboard/frontend/src/components/events/EventsTreeView.tsx index e46287752..d8e3d827d 100644 --- a/pkg/dashboard/frontend/src/components/events/EventsTreeView.tsx +++ b/pkg/dashboard/frontend/src/components/events/EventsTreeView.tsx @@ -1,5 +1,11 @@ import { type FC, useMemo } from 'react' -import type { Schedule, Subscriber, Topic } from '../../types' +import type { + BatchJob, + EventResource, + Schedule, + Subscriber, + Topic, +} from '../../types' import TreeView, { type TreeItemType } from '../shared/TreeView' import type { TreeItem, TreeItemIndex } from 'react-complex-tree' import { Tooltip, TooltipContent, TooltipTrigger } from '../ui/tooltip' @@ -10,10 +16,10 @@ import { getTopicSubscriptions } from '@/lib/utils/get-topic-subscriptions' export type EventsTreeItemType = TreeItemType interface Props { - resources: (Schedule | Topic)[] - onSelect: (resource: Schedule | Topic) => void - initialItem: Schedule | Topic - type: 'schedules' | 'topics' + resources: EventResource[] + onSelect: (resource: EventResource) => void + initialItem: EventResource + type: 'schedules' | 'topics' | 'jobs' subscriptions: Subscriber[] } diff --git a/pkg/dashboard/frontend/src/components/layout/AppLayout/AppLayout.tsx b/pkg/dashboard/frontend/src/components/layout/AppLayout/AppLayout.tsx index b64cdc641..94116d507 100644 --- a/pkg/dashboard/frontend/src/components/layout/AppLayout/AppLayout.tsx +++ b/pkg/dashboard/frontend/src/components/layout/AppLayout/AppLayout.tsx @@ -14,6 +14,7 @@ import { HeartIcon, CircleStackIcon, LockClosedIcon, + CpuChipIcon, } from '@heroicons/react/24/outline' import { cn } from '@/lib/utils' import { useWebSocket } from '../../../lib/hooks/use-web-socket' @@ -111,6 +112,11 @@ const AppLayout: React.FC = ({ href: '/', icon: GlobeAltIcon, }, + { + name: 'Batch Jobs', + href: '/jobs', + icon: CpuChipIcon, + }, { name: 'Databases', href: '/databases', diff --git a/pkg/dashboard/frontend/src/lib/hooks/use-history.ts b/pkg/dashboard/frontend/src/lib/hooks/use-history.ts index 7f1d0694f..07821db9f 100644 --- a/pkg/dashboard/frontend/src/lib/hooks/use-history.ts +++ b/pkg/dashboard/frontend/src/lib/hooks/use-history.ts @@ -24,9 +24,12 @@ export function useHistory(recordType: string) { ) const deleteHistory = useCallback(async () => { - const resp = await fetch(`http://${host}/api/history?type=${recordType}`, { - method: 'DELETE', - }) + const resp = await fetch( + `http://${host}/api/history?type=${recordType.toLowerCase()}`, + { + method: 'DELETE', + }, + ) if (resp.ok) { toast.success('Cleared History') diff --git a/pkg/dashboard/frontend/src/lib/utils/generate-architecture-data.ts b/pkg/dashboard/frontend/src/lib/utils/generate-architecture-data.ts index aae2d78ba..c3e5abc0e 100644 --- a/pkg/dashboard/frontend/src/lib/utils/generate-architecture-data.ts +++ b/pkg/dashboard/frontend/src/lib/utils/generate-architecture-data.ts @@ -18,6 +18,7 @@ import { ArrowsRightLeftIcon, QueueListIcon, LockClosedIcon, + CogIcon, } from '@heroicons/react/24/outline' import { MarkerType, @@ -56,9 +57,16 @@ import { SQLNode } from '@/components/architecture/nodes/SQLNode' import { SiPostgresql } from 'react-icons/si' import { unique } from 'radash' import { SecretNode } from '@/components/architecture/nodes/SecretNode' +import { JobNode } from '@/components/architecture/nodes/JobNode' +import { + BatchNode, + type BatchNodeData, +} from '@/components/architecture/nodes/BatchNode' export const nodeTypes = { api: APINode, + batch: BatchNode, + job: JobNode, bucket: BucketNode, schedule: ScheduleNode, topic: TopicNode, @@ -182,6 +190,7 @@ const actionVerbs = [ 'Enqueue', 'Dequeue', 'Access', + 'Submit', ] function verbFromNitricAction(action: string) { @@ -246,6 +255,40 @@ export function generateArchitectureData(data: WebSocketResponse): { nodes.push(node) }) + // Generate nodes from batch jobs + data.jobs.forEach((job) => { + const node = createNode(job, 'job', { + title: job.name, + resource: job, + icon: CogIcon, + description: '', + address: `${data.triggerAddress}/batches/${job.name}`, + }) + + nodes.push(node) + + const target = data.batchServices.find((b) => + job.requestingServices.includes(b.name), + ) + + if (target) { + edges.push({ + id: `e-${job.name}-${target.name}`, + source: node.id, + target: target.name, + animated: true, + markerEnd: { + type: MarkerType.ArrowClosed, + }, + markerStart: { + type: MarkerType.ArrowClosed, + orient: 'auto-start-reverse', + }, + label: 'Runs on', + }) + } + }) + // Generate nodes from websockets data.websockets.forEach((ws) => { const wsAddress = data.websocketAddresses[ws.name] @@ -526,6 +569,32 @@ export function generateArchitectureData(data: WebSocketResponse): { nodes.push(node) }) + data.batchServices.forEach((batch) => { + const node: Node = { + id: batch.name, + position: { x: 0, y: 0 }, + data: { + title: batch.name, + description: '', + resource: { + filePath: batch.filePath, + }, + icon: CpuChipIcon, + connectedEdges: [], + }, + type: 'batch', + } + + const connectedEdges = getConnectedEdges([node], edges) + node.data.connectedEdges = connectedEdges + node.data.description = + connectedEdges.length === 1 + ? `${connectedEdges.length} connection` + : `${connectedEdges.length} connections` + + nodes.push(node) + }) + if (import.meta.env.DEV) { console.log('nodes:', nodes) console.log('edges:', edges) diff --git a/pkg/dashboard/frontend/src/pages/jobs.astro b/pkg/dashboard/frontend/src/pages/jobs.astro new file mode 100644 index 000000000..da0eb4652 --- /dev/null +++ b/pkg/dashboard/frontend/src/pages/jobs.astro @@ -0,0 +1,8 @@ +--- +import EventExplorer from "../components/events/EventsExplorer"; +import Layout from "../layouts/Layout.astro"; +--- + + + + diff --git a/pkg/dashboard/frontend/src/types.ts b/pkg/dashboard/frontend/src/types.ts index c73ae9ce7..a180583e9 100644 --- a/pkg/dashboard/frontend/src/types.ts +++ b/pkg/dashboard/frontend/src/types.ts @@ -46,10 +46,15 @@ export type Topic = BaseResource export type Service = BaseResource +export type Batch = BaseResource + +export type BatchJob = BaseResource + export interface History { apis: ApiHistoryItem[] schedules: EventHistoryItem[] topics: EventHistoryItem[] + jobs: EventHistoryItem[] } export type WebsocketEvent = 'connect' | 'disconnect' | 'message' @@ -109,6 +114,8 @@ export interface WebSocketResponse { projectName: string buckets: Bucket[] apis: Api[] + batchServices: Batch[] + jobs: BatchJob[] schedules: Schedule[] notifications: Notification[] subscriptions: Subscriber[] @@ -182,7 +189,12 @@ export interface HistoryItem { event: T } -export type EventHistoryItem = TopicHistoryItem | ScheduleHistoryItem +export type EventHistoryItem = + | TopicHistoryItem + | ScheduleHistoryItem + | BatchHistoryItem + +export type EventResource = Schedule | Topic | BatchJob export type TopicHistoryItem = HistoryItem<{ name: string @@ -190,6 +202,12 @@ export type TopicHistoryItem = HistoryItem<{ success: boolean }> +export type BatchHistoryItem = HistoryItem<{ + name: string + payload: string + success: boolean +}> + export type ScheduleHistoryItem = HistoryItem<{ name: string success: boolean diff --git a/pkg/dashboard/handlers.go b/pkg/dashboard/handlers.go index eb1c6763e..58af86bdf 100644 --- a/pkg/dashboard/handlers.go +++ b/pkg/dashboard/handlers.go @@ -33,6 +33,7 @@ import ( "github.com/spf13/afero" "github.com/nitrictech/cli/pkg/cloud/apis" + "github.com/nitrictech/cli/pkg/cloud/batch" "github.com/nitrictech/cli/pkg/cloud/schedules" "github.com/nitrictech/cli/pkg/cloud/topics" "github.com/nitrictech/cli/pkg/cloud/websockets" @@ -587,3 +588,18 @@ func (d *Dashboard) handleSchedulesHistory(action schedules.ActionState) { log.Fatal(err) } } + +func (d *Dashboard) handleBatchJobsHistory(action batch.ActionState) { + err := d.writeHistoryRecord(&HistoryEvent[any]{ + Time: time.Now().UnixMilli(), + RecordType: BATCHJOBS, + Event: BatchHistoryItem{ + Name: action.JobName, + Payload: action.Payload, + Success: action.Success, + }, + }) + if err != nil { + log.Fatal(err) + } +} diff --git a/pkg/dashboard/history.go b/pkg/dashboard/history.go index d2a1acd78..5c0b2a039 100644 --- a/pkg/dashboard/history.go +++ b/pkg/dashboard/history.go @@ -32,14 +32,16 @@ type HistoryEvents struct { ScheduleHistory []*HistoryEvent[ScheduleHistoryItem] `json:"schedules"` TopicHistory []*HistoryEvent[TopicHistoryItem] `json:"topics"` ApiHistory []*HistoryEvent[ApiHistoryItem] `json:"apis"` + BatchHistory []*HistoryEvent[BatchHistoryItem] `json:"jobs"` } type RecordType string const ( - API RecordType = "apis" - TOPIC RecordType = "topics" - SCHEDULE RecordType = "schedules" + API RecordType = "apis" + TOPIC RecordType = "topics" + SCHEDULE RecordType = "schedules" + BATCHJOBS RecordType = "jobs" ) type HistoryItem interface { @@ -58,6 +60,12 @@ type TopicHistoryItem struct { Success bool `json:"success,omitempty"` } +type BatchHistoryItem struct { + Name string `json:"name,omitempty"` + Payload string `json:"payload,omitempty"` + Success bool `json:"success,omitempty"` +} + type ScheduleHistoryItem struct { Name string `json:"name,omitempty"` Success bool `json:"success,omitempty"` @@ -152,10 +160,16 @@ func (d *Dashboard) ReadAllHistoryRecords() (*HistoryEvents, error) { return nil, fmt.Errorf("error occurred reading api history: %w", err) } + jobs, err := ReadHistoryRecords[BatchHistoryItem](d.project.Directory, BATCHJOBS) + if err != nil { + return nil, fmt.Errorf("error occurred reading batch job history: %w", err) + } + return &HistoryEvents{ ScheduleHistory: schedules, TopicHistory: topics, ApiHistory: apis, + BatchHistory: jobs, }, nil } diff --git a/pkg/preview/feature.go b/pkg/preview/feature.go index 52bff95d8..ef6a3da82 100644 --- a/pkg/preview/feature.go +++ b/pkg/preview/feature.go @@ -22,4 +22,5 @@ const ( Feature_DockerProviders Feature = "docker-providers" Feature_BetaProviders Feature = "beta-providers" Feature_SqlDatabases Feature = "sql-databases" + Feature_BatchServices Feature = "batch-services" ) diff --git a/pkg/project/batch.go b/pkg/project/batch.go new file mode 100644 index 000000000..72733c9ea --- /dev/null +++ b/pkg/project/batch.go @@ -0,0 +1,388 @@ +// Copyright Nitric Pty Ltd. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package project + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + goruntime "runtime" + "strings" + "syscall" + + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/pkg/stdcopy" + "github.com/docker/go-connections/nat" + "github.com/samber/lo" + "github.com/spf13/afero" + + "github.com/nitrictech/cli/pkg/docker" + "github.com/nitrictech/cli/pkg/netx" + "github.com/nitrictech/cli/pkg/project/runtime" + "github.com/nitrictech/nitric/core/pkg/env" + "github.com/nitrictech/nitric/core/pkg/logger" +) + +type Batch struct { + Name string + + // filepath relative to the project root directory + basedir string + // filepath relative to the basedir + filepath string + buildContext runtime.RuntimeBuildContext + + runCmd string +} + +func (s *Batch) GetFilePath() string { + return filepath.Join(s.basedir, s.filepath) +} + +func (s *Batch) GetAbsoluteFilePath() (string, error) { + return filepath.Abs(s.GetFilePath()) +} + +// Run - runs the service using the provided command, typically not in a container. +func (s *Batch) Run(stop <-chan bool, updates chan<- ServiceRunUpdate, env map[string]string) error { + if s.runCmd == "" { + return fmt.Errorf("no start command provided for service %s", s.filepath) + } + + // this could be improve with real env var substitution. + startCmd := strings.ReplaceAll(s.runCmd, "$SERVICE_PATH", s.filepath) + startCmd = strings.ReplaceAll(startCmd, "${SERVICE_PATH}", s.filepath) + + if !strings.Contains(startCmd, s.filepath) { + logger.Warnf("Start cmd for service %s does not contain $SERVICE_PATH, check the service start configuration in nitric.yaml", s.filepath) + } + + commandParts := strings.Split(startCmd, " ") + cmd := exec.Command( + commandParts[0], + commandParts[1:]..., + ) + + cmd.Env = append([]string{}, os.Environ()...) + cmd.Dir = s.basedir + + for k, v := range env { + cmd.Env = append(cmd.Env, k+"="+v) + } + + cmd.Stdout = &ServiceRunUpdateWriter{ + updates: updates, + serviceName: s.Name, + label: s.GetFilePath(), + status: ServiceRunStatus_Running, + } + + cmd.Stderr = &ServiceRunUpdateWriter{ + updates: updates, + serviceName: s.Name, + label: s.GetFilePath(), + status: ServiceRunStatus_Error, + } + + errChan := make(chan error) + + go func() { + err := cmd.Start() + if err != nil { + errChan <- err + } else { + updates <- ServiceRunUpdate{ + ServiceName: s.Name, + Label: "nitric", + Status: ServiceRunStatus_Running, + Message: fmt.Sprintf("started service %s", s.GetFilePath()), + } + } + + err = cmd.Wait() + errChan <- err + }() + + go func(cmd *exec.Cmd) { + <-stop + + err := cmd.Process.Signal(syscall.SIGTERM) + if err != nil { + _ = cmd.Process.Kill() + } + }(cmd) + + err := <-errChan + updates <- ServiceRunUpdate{ + ServiceName: s.Name, + Status: ServiceRunStatus_Error, + Err: err, + } + + return err +} + +// RunContainer - Runs a container for the service, blocking until the container exits +func (s *Batch) RunContainer(stop <-chan bool, updates chan<- ServiceRunUpdate, opts ...RunContainerOption) error { + runtimeOptions := lo.ToPtr(defaultRunContainerOptions) + + for _, opt := range opts { + opt(runtimeOptions) + } + + dockerClient, err := docker.New() + if err != nil { + return err + } + + hostConfig := &container.HostConfig{ + // TODO: make this configurable through an cmd param + AutoRemove: true, + // LogConfig: *f.ce.Logger(f.runCtx).Config(), + LogConfig: container.LogConfig{ + Type: "json-file", + Config: map[string]string{ + "max-size": "10m", + "max-file": "3", + }, + }, + } + + if goruntime.GOOS == "linux" { + dockerHost := env.GetEnv("NITRIC_DOCKER_HOST", "172.17.0.1") + + // setup host.docker.internal to route to host gateway + // to access rpc server hosted by local CLI run + hostConfig.ExtraHosts = []string{"host.docker.internal:" + dockerHost.String()} + } + + randomPort, _ := netx.TakePort(1) + hostProxyPort := fmt.Sprint(randomPort[0]) + env := []string{ + fmt.Sprintf("NITRIC_ENVIRONMENT=%s", runtimeOptions.nitricEnvironment), + // FIXME: Ensure environment variable consistency in all SDKs, then remove duplicates here. + fmt.Sprintf("SERVICE_ADDRESS=%s", fmt.Sprintf("%s:%s", runtimeOptions.nitricHost, runtimeOptions.nitricPort)), + fmt.Sprintf("NITRIC_SERVICE_PORT=%s", runtimeOptions.nitricPort), + fmt.Sprintf("NITRIC_SERVICE_HOST=%s", runtimeOptions.nitricHost), + fmt.Sprintf("NITRIC_HTTP_PROXY_PORT=%d", randomPort[0]), + } + + for k, v := range runtimeOptions.envVars { + env = append(env, k+"="+v) + } + + hostConfig.PortBindings = nat.PortMap{ + nat.Port(hostProxyPort): []nat.PortBinding{ + { + HostPort: hostProxyPort, + }, + }, + } + + containerConfig := &container.Config{ + Image: s.Name, // Select an image to use based on the handler + Env: env, + ExposedPorts: nat.PortSet{ + nat.Port(hostProxyPort): struct{}{}, + }, + } + + // Create the container + containerId, err := dockerClient.ContainerCreate( + containerConfig, + hostConfig, + nil, + s.Name, + ) + if err != nil { + updates <- ServiceRunUpdate{ + ServiceName: s.Name, + Label: s.GetFilePath(), + Status: ServiceRunStatus_Error, + Err: err, + } + + return nil + } + + err = dockerClient.ContainerStart(context.TODO(), containerId, container.StartOptions{}) + if err != nil { + updates <- ServiceRunUpdate{ + ServiceName: s.Name, + Label: s.GetFilePath(), + Status: ServiceRunStatus_Error, + Err: err, + } + + return nil + } + + updates <- ServiceRunUpdate{ + ServiceName: s.Name, + Label: s.GetFilePath(), + Message: fmt.Sprintf("Service %s started", s.Name), + Status: ServiceRunStatus_Running, + } + + // Attach to the container to get stdout and stderr + attachOptions := container.AttachOptions{ + Stream: true, + Stdout: true, + Stderr: true, + } + + attachResponse, err := dockerClient.ContainerAttach(context.TODO(), containerId, attachOptions) + if err != nil { + return fmt.Errorf("error attaching to container %s: %w", s.Name, err) + } + + // Use a separate goroutine to handle the container's output + go func() { + defer attachResponse.Close() + // Using io.Copy to send the output to a writer + _, err := io.Copy(writerFunc(func(p []byte) (int, error) { + updates <- ServiceRunUpdate{ + ServiceName: s.Name, + Label: s.GetFilePath(), + Message: string(p), + Status: ServiceRunStatus_Running, + } + + return len(p), nil + }), attachResponse.Reader) + if err != nil { + updates <- ServiceRunUpdate{ + ServiceName: s.Name, + Label: s.GetFilePath(), + Status: ServiceRunStatus_Error, + Err: err, + } + } + }() + + okChan, errChan := dockerClient.ContainerWait(context.TODO(), containerId, container.WaitConditionNotRunning) + + for { + select { + case err := <-errChan: + updates <- ServiceRunUpdate{ + ServiceName: s.Name, + Label: s.GetFilePath(), + Err: err, + Status: ServiceRunStatus_Error, + } + + return err + case okBody := <-okChan: + if okBody.StatusCode != 0 { + logOptions := container.LogsOptions{ShowStdout: true, ShowStderr: true, Tail: "20"} + + logReader, err := dockerClient.ContainerLogs(context.Background(), containerId, logOptions) + if err != nil { + return err + } + + // Create a buffer to hold the logs + var logs bytes.Buffer + if _, err := stdcopy.StdCopy(&logs, &logs, logReader); err != nil { + return fmt.Errorf("error reading logs for service %s: %w", s.Name, err) + } + + err = fmt.Errorf("service %s exited with non 0 status\n %s", s.Name, logs.String()) + + updates <- ServiceRunUpdate{ + ServiceName: s.Name, + Label: s.GetFilePath(), + Err: err, + Status: ServiceRunStatus_Error, + } + + return err + } else { + updates <- ServiceRunUpdate{ + Label: s.GetFilePath(), + ServiceName: s.Name, + Message: "Service successfully exited", + Status: ServiceRunStatus_Done, + } + } + + return nil + case <-stop: + if err := dockerClient.ContainerStop(context.Background(), containerId, container.StopOptions{}); err != nil { + updates <- ServiceRunUpdate{ + Label: s.GetFilePath(), + ServiceName: s.Name, + Status: ServiceRunStatus_Error, + Err: err, + } + + return nil + } + } + } +} + +// FIXME: Duplicate code from service.go +func (s *Batch) BuildImage(fs afero.Fs, logs io.Writer) error { + dockerClient, err := docker.New() + if err != nil { + return err + } + + err = fs.MkdirAll(tempBuildDir, os.ModePerm) + if err != nil { + return fmt.Errorf("unable to create temporary build directory %s: %w", tempBuildDir, err) + } + + tmpDockerFile, err := afero.TempFile(fs, tempBuildDir, fmt.Sprintf("%s-*.dockerfile", s.Name)) + if err != nil { + return fmt.Errorf("unable to create temporary dockerfile for service %s: %w", s.Name, err) + } + + if err := afero.WriteFile(fs, tmpDockerFile.Name(), []byte(s.buildContext.DockerfileContents), os.ModePerm); err != nil { + return fmt.Errorf("unable to write temporary dockerfile for service %s: %w", s.Name, err) + } + + defer func() { + tmpDockerFile.Close() + + err := fs.Remove(tmpDockerFile.Name()) + if err != nil { + logger.Errorf("unable to remove temporary dockerfile %s: %s", tmpDockerFile.Name(), err) + } + }() + + // build the docker image + err = dockerClient.Build( + tmpDockerFile.Name(), + s.buildContext.BaseDirectory, + s.Name, + s.buildContext.BuildArguments, + strings.Split(s.buildContext.IgnoreFileContents, "\n"), + logs, + ) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/project/config.go b/pkg/project/config.go index 8669f0d1d..6e815f01c 100644 --- a/pkg/project/config.go +++ b/pkg/project/config.go @@ -36,7 +36,14 @@ type RuntimeConfiguration struct { Args map[string]string } -type ServiceConfiguration struct { +type BaseService interface { + GetBasedir() string + GetMatch() string + GetRuntime() string + GetStart() string +} + +type BaseServiceConfiguration struct { // The base directory for source files Basedir string `yaml:"basedir"` @@ -46,11 +53,35 @@ type ServiceConfiguration struct { // This is the custom runtime version (is custom if not nil, we auto-detect a standard language runtime) Runtime string `yaml:"runtime,omitempty"` + // This is a command that will be use to run these services when using nitric start + Start string `yaml:"start"` +} + +func (b BaseServiceConfiguration) GetBasedir() string { + return b.Basedir +} + +func (b BaseServiceConfiguration) GetMatch() string { + return b.Match +} + +func (b BaseServiceConfiguration) GetRuntime() string { + return b.Runtime +} + +func (b BaseServiceConfiguration) GetStart() string { + return b.Start +} + +type ServiceConfiguration struct { + BaseServiceConfiguration `yaml:",inline"` + // This allows specifying a particular service type (e.g. "Job"), this is optional and custom service types can be defined for each stack Type string `yaml:"type,omitempty"` +} - // This is a command that will be use to run these services when using nitric start - Start string `yaml:"start"` +type BatchConfiguration struct { + BaseServiceConfiguration `yaml:",inline"` } type ProjectConfiguration struct { @@ -58,6 +89,7 @@ type ProjectConfiguration struct { Directory string `yaml:"-"` Services []ServiceConfiguration `yaml:"services"` Ports map[string]int `yaml:"ports,omitempty"` + Batches []BatchConfiguration `yaml:"batch-services"` Runtimes map[string]RuntimeConfiguration `yaml:"runtimes,omitempty"` Preview []preview.Feature `yaml:"preview,omitempty"` } diff --git a/pkg/project/migrations.go b/pkg/project/migrations.go index 923f025eb..0937d3774 100644 --- a/pkg/project/migrations.go +++ b/pkg/project/migrations.go @@ -53,7 +53,7 @@ func migrationImageName(dbName string) string { func BuildAndRunMigrations(fs afero.Fs, servers map[string]*sql.DatabaseServer, databasesToMigrate map[string]*resourcespb.SqlDatabaseResource) error { serviceRequirements := collector.MakeDatabaseServiceRequirements(databasesToMigrate) - migrationImageContexts, err := collector.GetMigrationImageBuildContexts(serviceRequirements, fs) + migrationImageContexts, err := collector.GetMigrationImageBuildContexts(serviceRequirements, []*collector.BatchRequirements{}, fs) if err != nil { return fmt.Errorf("failed to get migration image build contexts: %w", err) } diff --git a/pkg/project/project.go b/pkg/project/project.go index 94e542cb8..f394ba037 100644 --- a/pkg/project/project.go +++ b/pkg/project/project.go @@ -44,17 +44,6 @@ import ( "github.com/nitrictech/cli/pkg/project/localconfig" "github.com/nitrictech/cli/pkg/project/runtime" "github.com/nitrictech/nitric/core/pkg/logger" - apispb "github.com/nitrictech/nitric/core/pkg/proto/apis/v1" - httppb "github.com/nitrictech/nitric/core/pkg/proto/http/v1" - kvstorepb "github.com/nitrictech/nitric/core/pkg/proto/kvstore/v1" - queuespb "github.com/nitrictech/nitric/core/pkg/proto/queues/v1" - resourcespb "github.com/nitrictech/nitric/core/pkg/proto/resources/v1" - schedulespb "github.com/nitrictech/nitric/core/pkg/proto/schedules/v1" - secretspb "github.com/nitrictech/nitric/core/pkg/proto/secrets/v1" - sqlpb "github.com/nitrictech/nitric/core/pkg/proto/sql/v1" - storagepb "github.com/nitrictech/nitric/core/pkg/proto/storage/v1" - topicspb "github.com/nitrictech/nitric/core/pkg/proto/topics/v1" - websocketspb "github.com/nitrictech/nitric/core/pkg/proto/websockets/v1" ) type Project struct { @@ -64,12 +53,79 @@ type Project struct { LocalConfig localconfig.LocalConfiguration services []Service + batches []Batch } func (p *Project) GetServices() []Service { return p.services } +func (p *Project) GetBatchServices() []Batch { + return p.batches +} + +// TODO: Reduce duplicate code +// BuildBatches - Builds all the batches in the project +func (p *Project) BuildBatches(fs afero.Fs) (chan ServiceBuildUpdate, error) { + updatesChan := make(chan ServiceBuildUpdate) + + if len(p.services) == 0 { + return nil, fmt.Errorf("no services found in project, nothing to build. This may indicate misconfigured `match` patterns in your nitric.yaml file") + } + + maxConcurrentBuilds := make(chan struct{}, min(goruntime.NumCPU(), goruntime.GOMAXPROCS(0))) + + waitGroup := sync.WaitGroup{} + + for _, batch := range p.batches { + waitGroup.Add(1) + // Create writer + serviceBuildUpdateWriter := &serviceBuildUpdateWriter{ + buildUpdateChan: updatesChan, + serviceName: batch.Name, + } + + go func(svc Batch, writer io.Writer) { + // Acquire a token by filling the maxConcurrentBuilds channel + // this will block once the buffer is full + maxConcurrentBuilds <- struct{}{} + + // Start goroutine + if err := svc.BuildImage(fs, writer); err != nil { + updatesChan <- ServiceBuildUpdate{ + ServiceName: svc.Name, + Err: err, + Message: err.Error(), + Status: ServiceBuildStatus_Error, + } + } else { + updatesChan <- ServiceBuildUpdate{ + ServiceName: svc.Name, + Message: "Build Complete", + Status: ServiceBuildStatus_Complete, + } + } + + // release our lock + <-maxConcurrentBuilds + + waitGroup.Done() + }(batch, serviceBuildUpdateWriter) + } + + go func() { + waitGroup.Wait() + // Drain the semaphore to make sure all goroutines have finished + for i := 0; i < cap(maxConcurrentBuilds); i++ { + maxConcurrentBuilds <- struct{}{} + } + + close(updatesChan) + }() + + return updatesChan, nil +} + // BuildServices - Builds all the services in the project func (p *Project) BuildServices(fs afero.Fs) (chan ServiceBuildUpdate, error) { updatesChan := make(chan ServiceBuildUpdate) @@ -134,19 +190,7 @@ func (p *Project) collectServiceRequirements(service Service) (*collector.Servic // start a grpc service with this registered grpcServer := grpc.NewServer() - resourcespb.RegisterResourcesServer(grpcServer, serviceRequirements) - apispb.RegisterApiServer(grpcServer, serviceRequirements.ApiServer) - schedulespb.RegisterSchedulesServer(grpcServer, serviceRequirements) - topicspb.RegisterTopicsServer(grpcServer, serviceRequirements) - topicspb.RegisterSubscriberServer(grpcServer, serviceRequirements) - websocketspb.RegisterWebsocketHandlerServer(grpcServer, serviceRequirements) - storagepb.RegisterStorageListenerServer(grpcServer, serviceRequirements) - httppb.RegisterHttpServer(grpcServer, serviceRequirements) - storagepb.RegisterStorageServer(grpcServer, serviceRequirements) - queuespb.RegisterQueuesServer(grpcServer, serviceRequirements) - kvstorepb.RegisterKvStoreServer(grpcServer, serviceRequirements) - sqlpb.RegisterSqlServer(grpcServer, serviceRequirements) - secretspb.RegisterSecretManagerServer(grpcServer, serviceRequirements) + serviceRequirements.RegisterServices(grpcServer) listener, err := net.Listen("tcp", ":") if err != nil { @@ -217,6 +261,59 @@ func (p *Project) collectServiceRequirements(service Service) (*collector.Servic return serviceRequirements, nil } +func (p *Project) collectBatchRequirements(service Batch) (*collector.BatchRequirements, error) { + serviceRequirements := collector.NewBatchRequirements(service.Name, service.GetFilePath()) + + // start a grpc service with this registered + grpcServer := grpc.NewServer() + + serviceRequirements.RegisterServices(grpcServer) + + listener, err := net.Listen("tcp", ":") + if err != nil { + return nil, err + } + + // register non-blocking + go func() { + err := grpcServer.Serve(listener) + if err != nil { + logger.Errorf("unable to start local Nitric collection server: %s", err) + } + }() + + defer grpcServer.Stop() + + // run the service we want to collect for targeting the grpc server + // TODO: load and run .env files, etc. + stopChannel := make(chan bool) + updatesChannel := make(chan ServiceRunUpdate) + + go func() { + for range updatesChannel { + // TODO: Provide some updates - bubbletea nice output + // fmt.Println("container update:", update) + continue + } + }() + + _, port, err := net.SplitHostPort(listener.Addr().String()) + if err != nil { + return nil, fmt.Errorf("unable to split host and port for local Nitric collection server: %w", err) + } + + err = service.RunContainer(stopChannel, updatesChannel, WithNitricPort(port), WithNitricEnvironment("build")) + if err != nil { + return nil, err + } + + if serviceRequirements.HasDatabases() && !slices.Contains(p.Preview, preview.Feature_SqlDatabases) { + return nil, fmt.Errorf("service %s requires a database, but the project does not have the 'sql-databases' preview feature enabled. Please add sql-databases to the preview field of your nitric.yaml file to enable this feature", service.filepath) + } + + return serviceRequirements, nil +} + func (p *Project) CollectServicesRequirements() ([]*collector.ServiceRequirements, error) { allServiceRequirements := []*collector.ServiceRequirements{} serviceErrors := []error{} @@ -259,6 +356,48 @@ func (p *Project) CollectServicesRequirements() ([]*collector.ServiceRequirement return allServiceRequirements, nil } +func (p *Project) CollectBatchRequirements() ([]*collector.BatchRequirements, error) { + allBatchRequirements := []*collector.BatchRequirements{} + batchErrors := []error{} + + reqLock := sync.Mutex{} + errorLock := sync.Mutex{} + wg := sync.WaitGroup{} + + for _, batch := range p.batches { + b := batch + + wg.Add(1) + + go func(s Batch) { + defer wg.Done() + + batchRequirements, err := p.collectBatchRequirements(s) + if err != nil { + errorLock.Lock() + defer errorLock.Unlock() + + batchErrors = append(batchErrors, err) + + return + } + + reqLock.Lock() + defer reqLock.Unlock() + + allBatchRequirements = append(allBatchRequirements, batchRequirements) + }(b) + } + + wg.Wait() + + if len(batchErrors) > 0 { + return nil, errors.Join(batchErrors...) + } + + return allBatchRequirements, nil +} + // DefaultMigrationImage - Returns the default migration image name for the project // Also returns ok if image is required or not func (p *Project) DefaultMigrationImage(fs afero.Fs) (string, bool) { @@ -302,6 +441,65 @@ func (p *Project) RunServicesWithCommand(localCloud *cloud.LocalCloud, stop <-ch return group.Wait() } +// RunBatchesWithCommand - Runs all the batches locally using a startup command +// use the stop channel to stop all running batches +func (p *Project) RunBatchesWithCommand(localCloud *cloud.LocalCloud, stop <-chan bool, updates chan<- ServiceRunUpdate, env map[string]string) error { + stopChannels := lo.FanOut[bool](len(p.batches), 1, stop) + + group, _ := errgroup.WithContext(context.TODO()) + + for i, service := range p.batches { + idx := i + svc := service + + // start the service with the given file reference from its projects CWD + group.Go(func() error { + port, err := localCloud.AddBatch(svc.GetFilePath()) + if err != nil { + return err + } + + envVariables := map[string]string{ + "PYTHONUNBUFFERED": "TRUE", // ensure all print statements print immediately for python + "NITRIC_ENVIRONMENT": "run", + "SERVICE_ADDRESS": "localhost:" + strconv.Itoa(port), + } + + for key, value := range env { + envVariables[key] = value + } + + return svc.Run(stopChannels[idx], updates, envVariables) + }) + } + + return group.Wait() +} + +// RunBatches - Runs all the batches as containers +// use the stop channel to stop all running batches +func (p *Project) RunBatches(localCloud *cloud.LocalCloud, stop <-chan bool, updates chan<- ServiceRunUpdate, env map[string]string) error { + stopChannels := lo.FanOut[bool](len(p.batches), 1, stop) + + group, _ := errgroup.WithContext(context.TODO()) + + for i, service := range p.batches { + idx := i + svc := service + + group.Go(func() error { + port, err := localCloud.AddBatch(svc.GetFilePath()) + if err != nil { + return err + } + + return svc.RunContainer(stopChannels[idx], updates, WithNitricPort(strconv.Itoa(port)), WithEnvVars(env)) + }) + } + + return group.Wait() +} + // RunServices - Runs all the services as containers // use the stop channel to stop all running services func (p *Project) RunServices(localCloud *cloud.LocalCloud, stop <-chan bool, updates chan<- ServiceRunUpdate, env map[string]string) error { @@ -344,11 +542,21 @@ func (pc *ProjectConfiguration) pathToNormalizedServiceName(servicePath string) // fromProjectConfiguration creates a new Instance of a nitric Project from a configuration files contents func fromProjectConfiguration(projectConfig *ProjectConfiguration, localConfig *localconfig.LocalConfiguration, fs afero.Fs) (*Project, error) { services := []Service{} + batches := []Batch{} matches := map[string]string{} + baseServices := []BaseService{} for _, serviceSpec := range projectConfig.Services { - serviceMatch := filepath.Join(serviceSpec.Basedir, serviceSpec.Match) + baseServices = append(baseServices, serviceSpec) + } + + for _, batchSpec := range projectConfig.Batches { + baseServices = append(baseServices, batchSpec) + } + + for _, baseService := range baseServices { + serviceMatch := filepath.Join(baseService.GetBasedir(), baseService.GetMatch()) files, err := afero.Glob(fs, serviceMatch) if err != nil { @@ -356,7 +564,7 @@ func fromProjectConfiguration(projectConfig *ProjectConfiguration, localConfig * } for _, f := range files { - relativeServiceEntrypointPath, _ := filepath.Rel(filepath.Join(projectConfig.Directory, serviceSpec.Basedir), f) + relativeServiceEntrypointPath, _ := filepath.Rel(filepath.Join(projectConfig.Directory, baseService.GetBasedir()), f) projectRelativeServiceFile := filepath.Join(projectConfig.Directory, f) serviceName := projectConfig.pathToNormalizedServiceName(projectRelativeServiceFile) @@ -367,18 +575,18 @@ func fromProjectConfiguration(projectConfig *ProjectConfiguration, localConfig * return file != f }) - if serviceSpec.Runtime != "" { + if baseService.GetRuntime() != "" { // We have a custom runtime - customRuntime, ok := projectConfig.Runtimes[serviceSpec.Runtime] + customRuntime, ok := projectConfig.Runtimes[baseService.GetRuntime()] if !ok { - return nil, fmt.Errorf("unable to find runtime %s", serviceSpec.Runtime) + return nil, fmt.Errorf("unable to find runtime %s", baseService.GetRuntime()) } buildContext, err = runtime.NewBuildContext( relativeServiceEntrypointPath, customRuntime.Dockerfile, // will default to the project directory if not set - lo.Ternary(customRuntime.Context != "", customRuntime.Context, serviceSpec.Basedir), + lo.Ternary(customRuntime.Context != "", customRuntime.Context, baseService.GetBasedir()), customRuntime.Args, otherEntryPointFiles, fs, @@ -390,7 +598,7 @@ func fromProjectConfiguration(projectConfig *ProjectConfiguration, localConfig * buildContext, err = runtime.NewBuildContext( relativeServiceEntrypointPath, "", - serviceSpec.Basedir, + baseService.GetBasedir(), map[string]string{}, otherEntryPointFiles, fs, @@ -401,23 +609,42 @@ func fromProjectConfiguration(projectConfig *ProjectConfiguration, localConfig * } if matches[f] != "" { - return nil, fmt.Errorf("service file %s matched by multiple patterns: %s and %s, services must only be matched by a single pattern", f, matches[f], serviceSpec.Match) + return nil, fmt.Errorf("service file %s matched by multiple patterns: %s and %s, services must only be matched by a single pattern", f, matches[f], baseService.GetMatch()) } - matches[f] = serviceSpec.Match + matches[f] = baseService.GetMatch() - relativeFilePath, err := filepath.Rel(serviceSpec.Basedir, f) + relativeFilePath, err := filepath.Rel(baseService.GetBasedir(), f) if err != nil { return nil, fmt.Errorf("unable to get relative file path for service %s: %w", f, err) } - newService := NewService(serviceName, serviceSpec.Type, relativeFilePath, *buildContext, serviceSpec.Start) + if svc, ok := baseService.(ServiceConfiguration); ok { + newService := Service{ + Name: serviceName, + filepath: relativeFilePath, + basedir: baseService.GetBasedir(), + buildContext: *buildContext, + Type: svc.Type, + startCmd: svc.Start, + } - if serviceSpec.Type == "" { - serviceSpec.Type = "default" - } + if svc.Type == "" { + svc.Type = "default" + } + + services = append(services, newService) + } else if batch, ok := baseService.(BatchConfiguration); ok { + newBatch := Batch{ + Name: serviceName, + basedir: batch.Basedir, + filepath: relativeFilePath, + buildContext: *buildContext, + runCmd: batch.Start, + } - services = append(services, *newService) + batches = append(batches, newBatch) + } } } @@ -426,13 +653,20 @@ func fromProjectConfiguration(projectConfig *ProjectConfiguration, localConfig * localConfig = &localconfig.LocalConfiguration{} } - return &Project{ + project := &Project{ Name: projectConfig.Name, Directory: projectConfig.Directory, Preview: projectConfig.Preview, LocalConfig: *localConfig, services: services, - }, nil + batches: batches, + } + + if len(project.batches) > 0 && !slices.Contains(project.Preview, preview.Feature_BatchServices) { + return nil, fmt.Errorf("project contains batch services, but the project does not have the 'batch-services' preview feature enabled. Please add batch-services to the preview field of your nitric.yaml file to enable this feature") + } + + return project, nil } // FromFile - Loads a nitric project from a nitric.yaml file diff --git a/pkg/project/stack/aws.config.yaml b/pkg/project/stack/aws.config.yaml index 52eec77f2..ee3fbf0a6 100644 --- a/pkg/project/stack/aws.config.yaml +++ b/pkg/project/stack/aws.config.yaml @@ -1,5 +1,5 @@ # The nitric provider to use -provider: nitric/aws@1.11.6 +provider: nitric/aws@1.14.0 # The target aws region to deploy to # See available regions: # https://docs.aws.amazon.com/general/latest/gr/lambda-service.html diff --git a/pkg/project/stack/awstf.config.yaml b/pkg/project/stack/awstf.config.yaml index cff0f4ec9..813dd7556 100644 --- a/pkg/project/stack/awstf.config.yaml +++ b/pkg/project/stack/awstf.config.yaml @@ -1,5 +1,5 @@ # The nitric provider to use -provider: nitric/awstf@1.11.6 +provider: nitric/awstf@1.14.0 # The target aws region to deploy to # See available regions: # https://docs.aws.amazon.com/general/latest/gr/lambda-service.html diff --git a/pkg/project/stack/azure.config.yaml b/pkg/project/stack/azure.config.yaml index 3e84cfac7..dc81320bf 100644 --- a/pkg/project/stack/azure.config.yaml +++ b/pkg/project/stack/azure.config.yaml @@ -1,7 +1,7 @@ # The provider to use and it's published version # See releases: # https://github.com/nitrictech/nitric/tags -provider: nitric/azure@1.11.6 +provider: nitric/azure@1.14.0 # The target Azure region to deploy to # See available regions: # https://azure.microsoft.com/en-us/explore/global-infrastructure/products-by-region/?products=container-apps diff --git a/pkg/project/stack/gcp.config.yaml b/pkg/project/stack/gcp.config.yaml index 3de591f5b..d5f04dcae 100644 --- a/pkg/project/stack/gcp.config.yaml +++ b/pkg/project/stack/gcp.config.yaml @@ -1,7 +1,7 @@ # The provider to use and it's published version # See releases: # https://github.com/nitrictech/nitric/tags -provider: nitric/gcp@1.11.6 +provider: nitric/gcp@1.14.0 # The target GCP region to deploy to # See available regions: diff --git a/pkg/project/stack/gcptf.config.yaml b/pkg/project/stack/gcptf.config.yaml index ea5cfc38c..bfeb103d9 100644 --- a/pkg/project/stack/gcptf.config.yaml +++ b/pkg/project/stack/gcptf.config.yaml @@ -1,7 +1,7 @@ # The provider to use and it's published version # See releases: # https://github.com/nitrictech/nitric/tags -provider: nitric/gcptf@1.11.6 +provider: nitric/gcptf@1.14.0 # The target GCP region to deploy to # See available regions: