From c4f7357ddc5c68cf7ccbd5d1b3f6be1df3b1b4b6 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Tue, 27 Feb 2024 12:57:22 +1100 Subject: [PATCH] deprecate: environment storage bytesused for kibused --- DEPRECATIONS.md | 2 + .../01-populate-api-data-lagoon-demo.gql | 18 +++++++ services/actions-handler/go.mod | 2 +- services/actions-handler/go.sum | 16 +----- .../actions-handler/handler/action_storage.go | 53 +++++++++++++------ .../20240223000000_bytesused_kibused.js | 21 ++++++++ services/api/src/mocks.js | 4 +- services/api/src/models/environment.ts | 24 ++++----- .../src/resources/environment/resolvers.ts | 21 ++++++-- services/api/src/typeDefs.js | 22 +++++++- .../internal/lagoonclient/schema.graphql | 14 +++-- 11 files changed, 145 insertions(+), 52 deletions(-) create mode 100644 services/api/database/migrations/20240223000000_bytesused_kibused.js diff --git a/DEPRECATIONS.md b/DEPRECATIONS.md index afc434f2ef..4108121a62 100644 --- a/DEPRECATIONS.md +++ b/DEPRECATIONS.md @@ -15,6 +15,8 @@ release link: https://github.com/uselagoon/lagoon/releases/tag/v2.18.0 * The standard drupal based tasks that Lagoon ships with (drush ....) have been flagged as deprecated and should not be used anymore. These will need to be replaced with [custom tasks](https://docs.lagoon.sh/using-lagoon-advanced/custom-tasks/). Example replacement tasks will be provided prior to their removal. * This release introduces a deprecation of the `setEnvironmentServices` mutation to updated services for an environment, it is being replaced with `addOrUpdateEnvironmentService` and `deleteEnvironmentService`. This is becaues the type is being refactored to support additional information, and eventually additional functionality. For now, the actions-handler service will still support the older `setEnvironmentServices` for backwards compatability for a short period to allow older versions of `lagoon-remote` to still work, but a new version of `lagoon-remote` will be available that will no longer provides the payload that the actions-handler uses. * The value for `registry` which was previously required by the `lagoon-core` chart is no longer required. If you are using this, you will need to add it under the new `unauthenticatedRegistry` setting when installing `lagoon-remote` in the `lagoon-build-deploy` section of your values file. If you aren't using an actual registry and have the example `disabled-only-use-harbor-via-deploy-controller.invalid` value, then you do not need to do anything except you can now remove the `registry` setting from your core values file. +* The environment storage return field named `bytesUsed` is deprecated. The actual value stored is `kibibytes`. A new return field called `kibUsed` exists and should be used, the returned data is the same and both fields are still returned. `bytesUsed` will be removed in a future release, make any adjustments now to use `kibUsed`. This will be a breaking change in a future release. +* `addOrUpdateEnvironmentStorage` is deprecated, `addOrUpdateStorageOnEnvironment` is the replacement to use as it supports the updated input value for `kibUsed`. `addOrUpdateEnvironmentStorage` will be completely removed in a future release. ### Lagoon v2.17.0 diff --git a/local-dev/api-data-watcher-pusher/api-data/01-populate-api-data-lagoon-demo.gql b/local-dev/api-data-watcher-pusher/api-data/01-populate-api-data-lagoon-demo.gql index da904bb039..3c2c4ac512 100644 --- a/local-dev/api-data-watcher-pusher/api-data/01-populate-api-data-lagoon-demo.gql +++ b/local-dev/api-data-watcher-pusher/api-data/01-populate-api-data-lagoon-demo.gql @@ -357,6 +357,24 @@ mutation PopulateApi { type } + UIProject1Environment1addStorage: addOrUpdateStorageOnEnvironment(input:{ + environment: 3 + persistentStorageClaim: "nginx" + kibUsed: 200000 + }) { + id + kibUsed + } + + UIProject1Environment1addStorage: addOrUpdateStorageOnEnvironment(input:{ + environment: 3 + persistentStorageClaim: "mariadb" + kibUsed: 200000 + }) { + id + kibUsed + } + UIProject1Environment1addFacts: addFacts( input: { facts: [ diff --git a/services/actions-handler/go.mod b/services/actions-handler/go.mod index e64c1868a5..eff685ffad 100644 --- a/services/actions-handler/go.mod +++ b/services/actions-handler/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/cheshir/go-mq/v2 v2.0.1 - github.com/uselagoon/machinery v0.0.17-0.20240108054822-78639cc0a1f3 + github.com/uselagoon/machinery v0.0.17 gopkg.in/matryer/try.v1 v1.0.0-20150601225556-312d2599e12e ) diff --git a/services/actions-handler/go.sum b/services/actions-handler/go.sum index 5ba1324719..49235d16ed 100644 --- a/services/actions-handler/go.sum +++ b/services/actions-handler/go.sum @@ -839,20 +839,8 @@ github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/uselagoon/machinery v0.0.17-0.20240108020517-e6622621374c h1:xEUsDgNpM3ZaD0S6dsxQ+j/8hrnMM1HLEhbDUxrqFHE= -github.com/uselagoon/machinery v0.0.17-0.20240108020517-e6622621374c/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU= -github.com/uselagoon/machinery v0.0.17-0.20240108024302-9784c2db7f1b h1:eF3O0RMETo+Bk3/I1SLjwkTvqbn/SkwCyTPWOFAl6ek= -github.com/uselagoon/machinery v0.0.17-0.20240108024302-9784c2db7f1b/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU= -github.com/uselagoon/machinery v0.0.17-0.20240108025245-bf15fc15ab08 h1:SRCSGYkw6G77d20f5aA0Q/TJYhczybgEU4g7SodVsNg= -github.com/uselagoon/machinery v0.0.17-0.20240108025245-bf15fc15ab08/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU= -github.com/uselagoon/machinery v0.0.17-0.20240108043534-c066518389be h1:cOBkrQa7DYDnIVZebJViIc9XEggFKYjfSKzqHC9VckE= -github.com/uselagoon/machinery v0.0.17-0.20240108043534-c066518389be/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU= -github.com/uselagoon/machinery v0.0.17-0.20240108043655-6cf0c0b42884 h1:33GmXqKEkpPzsspIse3xx/aN/Leh+Jdl9QcRrbnEktg= -github.com/uselagoon/machinery v0.0.17-0.20240108043655-6cf0c0b42884/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU= -github.com/uselagoon/machinery v0.0.17-0.20240108050446-30ff0a7df794 h1:2LP/ytk7sY6BrVY67PizVVYF6EnZKPzwKEOJg8JFY1I= -github.com/uselagoon/machinery v0.0.17-0.20240108050446-30ff0a7df794/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU= -github.com/uselagoon/machinery v0.0.17-0.20240108054822-78639cc0a1f3 h1:DYklzy44C1s1a1O6LqAi8RUpuqDzTzJTnW9IRQ8J91k= -github.com/uselagoon/machinery v0.0.17-0.20240108054822-78639cc0a1f3/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU= +github.com/uselagoon/machinery v0.0.17 h1:rykgkboGvSX+aMa6MPXtR22DJ67eTPC4WITwpVDuj+Y= +github.com/uselagoon/machinery v0.0.17/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= diff --git a/services/actions-handler/handler/action_storage.go b/services/actions-handler/handler/action_storage.go index e2c25e507c..ee9ae94348 100644 --- a/services/actions-handler/handler/action_storage.go +++ b/services/actions-handler/handler/action_storage.go @@ -23,7 +23,8 @@ type Storage struct { type StorageClaim struct { Environment int `json:"environment"` PersisteStorageClaim string `json:"persistentStorageClaim"` - BytesUsed int `json:"bytesUsed"` + BytesUsed uint64 `json:"bytesUsed"` + KiBUsed uint64 `json:"kibUsed"` } func (m *Messenger) handleUpdateStorage(ctx context.Context, messageQueue *mq.MessageQueue, action *Action, messageID string) error { @@ -35,32 +36,54 @@ func (m *Messenger) handleUpdateStorage(ctx context.Context, messageQueue *mq.Me if err != nil { // the token wasn't generated if m.EnableDebug { - log.Println(fmt.Sprintf("%sERROR: unable to generate token: %v", prefix, err)) + log.Printf("%sERROR: unable to generate token: %v", prefix, err) } return nil } // the action data can contain multiple storage claims, so iterate over them here var errs []error for _, sc := range storageClaims.Claims { - sci := schema.UpdateEnvironmentStorageInput{} - scdata, _ := json.Marshal(sc) - json.Unmarshal(scdata, &sci) l := lclient.New(m.LagoonAPI.Endpoint, "actions-handler", &token, false) - environment, err := lagoon.UpdateStorage(ctx, &sci, l) - if err != nil { + var envID int + var hasErr error + // if this is a newer storage calculator with the `kibUsed` field, use the new mutation + if sc.KiBUsed != 0 { + scoei := schema.UpdateStorageOnEnvironmentInput{} + scdata, _ := json.Marshal(sc) + json.Unmarshal(scdata, &scoei) + environment, err := lagoon.UpdateStorageOnEnvironment(ctx, &scoei, l) + if err != nil { + hasErr = err + } else { + envID = environment.ID + } + } else { + // else use the old mutation + // @DEPRECATED to be removed in a future release + sci := schema.UpdateEnvironmentStorageInput{} + scdata, _ := json.Marshal(sc) + json.Unmarshal(scdata, &sci) + environment, err := lagoon.UpdateStorage(ctx, &sci, l) + if err != nil { + hasErr = err + } else { + envID = environment.ID + } + } + if hasErr != nil { // send the log to the lagoon-logs exchange to be processed m.toLagoonLogs(messageQueue, map[string]interface{}{ "severity": "error", "event": fmt.Sprintf("actions-handler:%s:error", action.EventType), - "meta": sci, - "message": err.Error(), + "meta": sc, + "message": hasErr.Error(), }) if m.EnableDebug { - log.Println(fmt.Sprintf("%sERROR: unable to update storage for environment %v in the api: %v", prefix, sci.Environment, err)) + log.Printf("%sERROR: unable to update storage for environment %v in the api: %v", prefix, sc.Environment, hasErr) } // if the error is in LagoonAPIErrorCheck, this should be retried - if LagoonAPIRetryErrorCheck(err) == nil { - errs = append(errs, err) + if LagoonAPIRetryErrorCheck(hasErr) == nil { + errs = append(errs, hasErr) } // try and update the next storage claim if there is one continue @@ -69,11 +92,11 @@ func (m *Messenger) handleUpdateStorage(ctx context.Context, messageQueue *mq.Me m.toLagoonLogs(messageQueue, map[string]interface{}{ "severity": "info", "event": fmt.Sprintf("actions-handler:%s:updated", action.EventType), - "meta": sci, - "message": fmt.Sprintf("updated environment: %v, storage claim: %s, id: %v", sci.Environment, sci.PersisteStorageClaim, environment.ID), + "meta": sc, + "message": fmt.Sprintf("updated environment: %v, storage claim: %s, id: %v", sc.Environment, sc.PersisteStorageClaim, envID), }) if m.EnableDebug { - log.Println(fmt.Sprintf("%supdated environment: %v, storage claim: %s, id: %v", prefix, sci.Environment, sci.PersisteStorageClaim, environment.ID)) + log.Printf("%supdated environment: %v, storage claim: %s, id: %v", prefix, sc.Environment, sc.PersisteStorageClaim, envID) } } if len(errs) > 0 { diff --git a/services/api/database/migrations/20240223000000_bytesused_kibused.js b/services/api/database/migrations/20240223000000_bytesused_kibused.js new file mode 100644 index 0000000000..dec6c35c63 --- /dev/null +++ b/services/api/database/migrations/20240223000000_bytesused_kibused.js @@ -0,0 +1,21 @@ +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.up = async function(knex) { + return knex.schema + .alterTable('environment_storage', function (table) { + table.renameColumn('bytes_used', 'kib_used'); + }) +}; + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.down = async function(knex) { + return knex.schema + .alterTable('environment_storage', function (table) { + table.renameColumn('kib_used', 'bytes_used'); + }) +}; \ No newline at end of file diff --git a/services/api/src/mocks.js b/services/api/src/mocks.js index 1c157567ce..a7e62257c7 100644 --- a/services/api/src/mocks.js +++ b/services/api/src/mocks.js @@ -353,7 +353,7 @@ mocks.EnvironmentStorage = (parent, args = {}, context, info) => ({ id: faker.random.number(), environment: args.hasOwnProperty('environment') ? { id: args.environment.id } : { id: mocks.Environment().id }, persistentStorageClaim: '', - bytesUsed: faker.random.number({ precision: 0.001 }), // Float + kibUsed: faker.random.number({ precision: 0.001 }), // Float updated: mocks.Date(), }); @@ -361,7 +361,7 @@ mocks.EnvironmentStorageMonth = () => { const date = new Date(mocks.Date()); return { month: `${date.getFullYear()}-${date.getMonth() + 1}`, - bytesUsed: faker.random.number({ precision: 0.001 }), + kibUsed: faker.random.number({ precision: 0.001 }), }; }; diff --git a/services/api/src/models/environment.ts b/services/api/src/models/environment.ts index e5af1f4b4c..c43b12592b 100644 --- a/services/api/src/models/environment.ts +++ b/services/api/src/models/environment.ts @@ -74,18 +74,18 @@ export const EnvironmentModel = (clients: { sqlClientPool: Pool }) => { }; const environmentStorageMonthByEnvironmentId = async (eid, month) => { - const str = ` - SELECT - SUM(bytes_used) as bytes_used, max(DATE_FORMAT(updated, '%Y-%m')) as month - FROM - environment_storage - WHERE - environment = :eid - AND YEAR(updated) = YEAR(STR_TO_DATE(:month, '%Y-%m')) - AND MONTH(updated) = MONTH(STR_TO_DATE(:month, '%Y-%m')) - `; - - const rows = await query(sqlClientPool, str, { eid, month }); + let q = knex('environment_storage') + .select(knex.raw('SUM(kib_used) as kib_used')) + .select(knex.raw('SUM(kib_used) as bytes_used')) // @DEPRECATE when `bytesUsed` is completely removed, this can be removed + .select(knex.raw(`max(DATE_FORMAT(updated, '%Y-%m')) as month`)) + .where('environment', eid) + .andWhere(knex.raw(`YEAR(updated) = YEAR(STR_TO_DATE(?, '%Y-%m'))`, month)) + .andWhere(knex.raw(`MONTH(updated) = MONTH(STR_TO_DATE(?, '%Y-%m'))`, month)) + + const rows = await query(sqlClientPool, q.toString()); + + rows.map(row => ({ ...row, bytesUsed: row.kibUsed})); // @DEPRECATE when `bytesUsed` is completely removed, this can be removed + return rows[0]; }; diff --git a/services/api/src/resources/environment/resolvers.ts b/services/api/src/resources/environment/resolvers.ts index cab7fb4a9c..7dc8107c6f 100644 --- a/services/api/src/resources/environment/resolvers.ts +++ b/services/api/src/resources/environment/resolvers.ts @@ -157,7 +157,9 @@ export const getEnvironmentStorageByEnvironmentId: ResolverFn = async ( const rows = await query(sqlClientPool, Sql.selectEnvironmentStorageByEnvironmentId(eid)) - return rows; + // @DEPRECATE when `bytesUsed` is completely removed, this can be reverted + return rows.map(row => ({ ...row, bytesUsed: row.kibUsed})); + // return rows; }; export const getEnvironmentStorageMonthByEnvironmentId: ResolverFn = async ( @@ -439,11 +441,22 @@ export const addOrUpdateEnvironmentStorage: ResolverFn = async ( : convertDateToMYSQLDateFormat(new Date().toISOString()) }; + + // @DEPRECATE when `bytesUsed` is completely removed, this block can be removed + if (input.kibUsed) { + // remove the bytesUsed input if kilobytes is provided + delete input.bytesUsed + } else { + // else set kibUsed to the old required input, then remove the old input + input.kibUsed = input.bytesUsed + delete input.bytesUsed + } + const createOrUpdateSql = knex('environment_storage') .insert(input) .onConflict('id') .merge({ - bytesUsed: input.bytesUsed + kibUsed: input.kibUsed }).toString(); const { insertId } = await query( @@ -459,7 +472,9 @@ export const addOrUpdateEnvironmentStorage: ResolverFn = async ( .toString() ); - const environment = R.path([0], rows); + // @DEPRECATE when `bytesUsed` is completely removed, this can be reverted + const environment = R.path([0], rows.map(row => ({ ...row, bytesUsed: row.kibUsed}))); + // const environment = R.path([0], rows); const { name: projectName } = await projectHelpers(sqlClientPool).getProjectByEnvironmentId(environment['environment']); userActivityLogger(`User updated environment storage on project '${projectName}'`, { diff --git a/services/api/src/typeDefs.js b/services/api/src/typeDefs.js index 835ec5d846..5b13bcce93 100644 --- a/services/api/src/typeDefs.js +++ b/services/api/src/typeDefs.js @@ -950,13 +950,15 @@ const typeDefs = gql` id: Int environment: Environment persistentStorageClaim: String - bytesUsed: Float + bytesUsed: Float @deprecated(reason: "The value of this is kibibytes, use kibUsed instead. This will be removed in a future release.") + kibUsed: Float updated: String } type EnvironmentStorageMonth { month: String - bytesUsed: Float + bytesUsed: Float @deprecated(reason: "The value of this is kibibytes, use kibUsed instead. This will be removed in a future release.") + kibUsed: Float } type EnvironmentHoursMonth { @@ -1596,6 +1598,19 @@ const typeDefs = gql` updated: String } + input AddOrUpdateStorageOnEnvironmentInput { + environment: Int! + persistentStorageClaim: String! + """ + kibUsed is a float to allow for greater than 32-bit integer inputs + """ + kibUsed: Float! + """ + Date in format 'YYYY-MM-DD' + """ + updated: String + } + input AddBackupInput { id: Int environment: Int! @@ -2320,6 +2335,9 @@ const typeDefs = gql` """ addOrUpdateEnvironmentStorage( input: AddOrUpdateEnvironmentStorageInput! + ): EnvironmentStorage @deprecated(reason: "Use addOrUpdateStorageonEnvironment instead") + addOrUpdateStorageonEnvironment( + input: AddOrUpdateStorageOnEnvironmentInput! ): EnvironmentStorage addNotificationSlack(input: AddNotificationSlackInput!): NotificationSlack updateNotificationSlack( diff --git a/services/workflows/internal/lagoonclient/schema.graphql b/services/workflows/internal/lagoonclient/schema.graphql index d28d5ce864..c951d05ce9 100644 --- a/services/workflows/internal/lagoonclient/schema.graphql +++ b/services/workflows/internal/lagoonclient/schema.graphql @@ -194,8 +194,14 @@ input AddOpenshiftInput { input AddOrUpdateEnvironmentStorageInput { environment: Int! persistentStorageClaim: String! + """ + @deprecated(reason: "When bytesUsed is removed, kibUsed will become a required field") + """ bytesUsed: Int! - + """ + kibUsed is a float to allow for greater than 32-bit integer inputs + """ + kibUsed: Float """ Date in format 'YYYY-MM-DD' """ @@ -811,13 +817,15 @@ type EnvironmentStorage { id: Int environment: Environment persistentStorageClaim: String - bytesUsed: Float + bytesUsed: Float @deprecated(reason: "The value of this is kibibytes, use kibUsed field instead") + kibUsed: Float updated: String } type EnvironmentStorageMonth { month: String - bytesUsed: Float + bytesUsed: Float @deprecated(reason: "The value of this is kibibytes, use kibUsed field instead") + kibUsed: Float } type EnvKeyValue {