diff --git a/DEPRECATIONS.md b/DEPRECATIONS.md index afc434f2ef..7460528dff 100644 --- a/DEPRECATIONS.md +++ b/DEPRECATIONS.md @@ -15,6 +15,8 @@ release link: https://github.com/uselagoon/lagoon/releases/tag/v2.18.0 * The standard drupal based tasks that Lagoon ships with (drush ....) have been flagged as deprecated and should not be used anymore. These will need to be replaced with [custom tasks](https://docs.lagoon.sh/using-lagoon-advanced/custom-tasks/). Example replacement tasks will be provided prior to their removal. * This release introduces a deprecation of the `setEnvironmentServices` mutation to updated services for an environment, it is being replaced with `addOrUpdateEnvironmentService` and `deleteEnvironmentService`. This is becaues the type is being refactored to support additional information, and eventually additional functionality. For now, the actions-handler service will still support the older `setEnvironmentServices` for backwards compatability for a short period to allow older versions of `lagoon-remote` to still work, but a new version of `lagoon-remote` will be available that will no longer provides the payload that the actions-handler uses. * The value for `registry` which was previously required by the `lagoon-core` chart is no longer required. If you are using this, you will need to add it under the new `unauthenticatedRegistry` setting when installing `lagoon-remote` in the `lagoon-build-deploy` section of your values file. If you aren't using an actual registry and have the example `disabled-only-use-harbor-via-deploy-controller.invalid` value, then you do not need to do anything except you can now remove the `registry` setting from your core values file. +* The environment storage return field named `bytesUsed` is deprecated. The actual value stored is `kibibytes`. A new return field called `kibUsed` exists and should be used, the returned data is the same and both fields are still returned. `bytesUsed` will be removed in a future release, make any adjustments now to use `kibUsed`. This will be a breaking change in a future release. +* When adding or updating environment storage using `addOrUpdateEnvironmentStorage`, the `bytesUsed` input is still a required field, `kibUsed` is optional. If `kibUsed` is optionally provided, the value of it will be preferred over `bytesUsed`. `bytesUsed` will be completely removed in a future release, `kibUsed` will be the required field. This will be a breaking change in a future release. ### Lagoon v2.17.0 diff --git a/local-dev/api-data-watcher-pusher/api-data/01-populate-api-data-lagoon-demo.gql b/local-dev/api-data-watcher-pusher/api-data/01-populate-api-data-lagoon-demo.gql index da904bb039..ea6e3305ed 100644 --- a/local-dev/api-data-watcher-pusher/api-data/01-populate-api-data-lagoon-demo.gql +++ b/local-dev/api-data-watcher-pusher/api-data/01-populate-api-data-lagoon-demo.gql @@ -357,6 +357,28 @@ mutation PopulateApi { type } + UIProject1Environment1addStorage: addOrUpdateEnvironmentStorage(input:{ + environment: 3 + persistentStorageClaim: "nginx" + bytesUsed: 200000 + kibUsed: 200000 + }) { + id + bytesUsed + kibUsed + } + + UIProject1Environment1addStorage: addOrUpdateEnvironmentStorage(input:{ + environment: 3 + persistentStorageClaim: "mariadb" + bytesUsed: 200000 + kibUsed: 200000 + }) { + id + bytesUsed + kibUsed + } + UIProject1Environment1addFacts: addFacts( input: { facts: [ diff --git a/services/actions-handler/go.mod b/services/actions-handler/go.mod index e64c1868a5..f3b794b162 100644 --- a/services/actions-handler/go.mod +++ b/services/actions-handler/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/cheshir/go-mq/v2 v2.0.1 - github.com/uselagoon/machinery v0.0.17-0.20240108054822-78639cc0a1f3 + github.com/uselagoon/machinery v0.0.17-0.20240227042958-7b4006aaf6cd gopkg.in/matryer/try.v1 v1.0.0-20150601225556-312d2599e12e ) diff --git a/services/actions-handler/go.sum b/services/actions-handler/go.sum index 5ba1324719..a9fb635777 100644 --- a/services/actions-handler/go.sum +++ b/services/actions-handler/go.sum @@ -853,6 +853,16 @@ github.com/uselagoon/machinery v0.0.17-0.20240108050446-30ff0a7df794 h1:2LP/ytk7 github.com/uselagoon/machinery v0.0.17-0.20240108050446-30ff0a7df794/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU= github.com/uselagoon/machinery v0.0.17-0.20240108054822-78639cc0a1f3 h1:DYklzy44C1s1a1O6LqAi8RUpuqDzTzJTnW9IRQ8J91k= github.com/uselagoon/machinery v0.0.17-0.20240108054822-78639cc0a1f3/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU= +github.com/uselagoon/machinery v0.0.17-0.20240222230758-208090a0bc18 h1:hIX+Fx5Or5nQ1pNTvX7QBIhmDNq9QCcVATCZ+RxSgwI= +github.com/uselagoon/machinery v0.0.17-0.20240222230758-208090a0bc18/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU= +github.com/uselagoon/machinery v0.0.17-0.20240222231146-70b5b91a3632 h1:k+VKnVqpH3LB574QFoYS5oiiftspkNlIwxurqnsWPpM= +github.com/uselagoon/machinery v0.0.17-0.20240222231146-70b5b91a3632/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU= +github.com/uselagoon/machinery v0.0.17-0.20240223003225-61f03325cde5 h1:W24cLoV/iuwmbpBm4HGbc6M79vS2AM06aIeffaIBrNs= +github.com/uselagoon/machinery v0.0.17-0.20240223003225-61f03325cde5/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU= +github.com/uselagoon/machinery v0.0.17-0.20240223004536-adb1b1a6dd8c h1:lCQf1N54gLFFKIw8x770BZCP+vgf+kEauNwn8i80kio= +github.com/uselagoon/machinery v0.0.17-0.20240223004536-adb1b1a6dd8c/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU= +github.com/uselagoon/machinery v0.0.17-0.20240227042958-7b4006aaf6cd h1:YN8DtIXYqPL88OYM4NRTeAiKo+WJr1K6UraLEdJYGrQ= +github.com/uselagoon/machinery v0.0.17-0.20240227042958-7b4006aaf6cd/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= diff --git a/services/actions-handler/handler/action_storage.go b/services/actions-handler/handler/action_storage.go index e2c25e507c..caaaa0a168 100644 --- a/services/actions-handler/handler/action_storage.go +++ b/services/actions-handler/handler/action_storage.go @@ -23,7 +23,8 @@ type Storage struct { type StorageClaim struct { Environment int `json:"environment"` PersisteStorageClaim string `json:"persistentStorageClaim"` - BytesUsed int `json:"bytesUsed"` + BytesUsed uint64 `json:"bytesUsed"` + KiBUsed uint64 `json:"kibUsed"` } func (m *Messenger) handleUpdateStorage(ctx context.Context, messageQueue *mq.MessageQueue, action *Action, messageID string) error { @@ -43,6 +44,13 @@ func (m *Messenger) handleUpdateStorage(ctx context.Context, messageQueue *mq.Me var errs []error for _, sc := range storageClaims.Claims { sci := schema.UpdateEnvironmentStorageInput{} + + // @DEPRECATE when `bytesUsed` is completely removed, this can be removed + // it just ensures that the `bytesUsed` value is always populated in case only `kibUsed` is provided from the storage-calculator + if sc.KiBUsed != 0 && sc.BytesUsed == 0 { + sc.BytesUsed = sc.KiBUsed + } + scdata, _ := json.Marshal(sc) json.Unmarshal(scdata, &sci) l := lclient.New(m.LagoonAPI.Endpoint, "actions-handler", &token, false) diff --git a/services/api/database/migrations/20240223000000_bytesused_kibused.js b/services/api/database/migrations/20240223000000_bytesused_kibused.js new file mode 100644 index 0000000000..dec6c35c63 --- /dev/null +++ b/services/api/database/migrations/20240223000000_bytesused_kibused.js @@ -0,0 +1,21 @@ +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.up = async function(knex) { + return knex.schema + .alterTable('environment_storage', function (table) { + table.renameColumn('bytes_used', 'kib_used'); + }) +}; + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.down = async function(knex) { + return knex.schema + .alterTable('environment_storage', function (table) { + table.renameColumn('kib_used', 'bytes_used'); + }) +}; \ No newline at end of file diff --git a/services/api/src/mocks.js b/services/api/src/mocks.js index 1c157567ce..a7e62257c7 100644 --- a/services/api/src/mocks.js +++ b/services/api/src/mocks.js @@ -353,7 +353,7 @@ mocks.EnvironmentStorage = (parent, args = {}, context, info) => ({ id: faker.random.number(), environment: args.hasOwnProperty('environment') ? { id: args.environment.id } : { id: mocks.Environment().id }, persistentStorageClaim: '', - bytesUsed: faker.random.number({ precision: 0.001 }), // Float + kibUsed: faker.random.number({ precision: 0.001 }), // Float updated: mocks.Date(), }); @@ -361,7 +361,7 @@ mocks.EnvironmentStorageMonth = () => { const date = new Date(mocks.Date()); return { month: `${date.getFullYear()}-${date.getMonth() + 1}`, - bytesUsed: faker.random.number({ precision: 0.001 }), + kibUsed: faker.random.number({ precision: 0.001 }), }; }; diff --git a/services/api/src/models/environment.ts b/services/api/src/models/environment.ts index e5af1f4b4c..c43b12592b 100644 --- a/services/api/src/models/environment.ts +++ b/services/api/src/models/environment.ts @@ -74,18 +74,18 @@ export const EnvironmentModel = (clients: { sqlClientPool: Pool }) => { }; const environmentStorageMonthByEnvironmentId = async (eid, month) => { - const str = ` - SELECT - SUM(bytes_used) as bytes_used, max(DATE_FORMAT(updated, '%Y-%m')) as month - FROM - environment_storage - WHERE - environment = :eid - AND YEAR(updated) = YEAR(STR_TO_DATE(:month, '%Y-%m')) - AND MONTH(updated) = MONTH(STR_TO_DATE(:month, '%Y-%m')) - `; - - const rows = await query(sqlClientPool, str, { eid, month }); + let q = knex('environment_storage') + .select(knex.raw('SUM(kib_used) as kib_used')) + .select(knex.raw('SUM(kib_used) as bytes_used')) // @DEPRECATE when `bytesUsed` is completely removed, this can be removed + .select(knex.raw(`max(DATE_FORMAT(updated, '%Y-%m')) as month`)) + .where('environment', eid) + .andWhere(knex.raw(`YEAR(updated) = YEAR(STR_TO_DATE(?, '%Y-%m'))`, month)) + .andWhere(knex.raw(`MONTH(updated) = MONTH(STR_TO_DATE(?, '%Y-%m'))`, month)) + + const rows = await query(sqlClientPool, q.toString()); + + rows.map(row => ({ ...row, bytesUsed: row.kibUsed})); // @DEPRECATE when `bytesUsed` is completely removed, this can be removed + return rows[0]; }; diff --git a/services/api/src/resources/environment/resolvers.ts b/services/api/src/resources/environment/resolvers.ts index cab7fb4a9c..7dc8107c6f 100644 --- a/services/api/src/resources/environment/resolvers.ts +++ b/services/api/src/resources/environment/resolvers.ts @@ -157,7 +157,9 @@ export const getEnvironmentStorageByEnvironmentId: ResolverFn = async ( const rows = await query(sqlClientPool, Sql.selectEnvironmentStorageByEnvironmentId(eid)) - return rows; + // @DEPRECATE when `bytesUsed` is completely removed, this can be reverted + return rows.map(row => ({ ...row, bytesUsed: row.kibUsed})); + // return rows; }; export const getEnvironmentStorageMonthByEnvironmentId: ResolverFn = async ( @@ -439,11 +441,22 @@ export const addOrUpdateEnvironmentStorage: ResolverFn = async ( : convertDateToMYSQLDateFormat(new Date().toISOString()) }; + + // @DEPRECATE when `bytesUsed` is completely removed, this block can be removed + if (input.kibUsed) { + // remove the bytesUsed input if kilobytes is provided + delete input.bytesUsed + } else { + // else set kibUsed to the old required input, then remove the old input + input.kibUsed = input.bytesUsed + delete input.bytesUsed + } + const createOrUpdateSql = knex('environment_storage') .insert(input) .onConflict('id') .merge({ - bytesUsed: input.bytesUsed + kibUsed: input.kibUsed }).toString(); const { insertId } = await query( @@ -459,7 +472,9 @@ export const addOrUpdateEnvironmentStorage: ResolverFn = async ( .toString() ); - const environment = R.path([0], rows); + // @DEPRECATE when `bytesUsed` is completely removed, this can be reverted + const environment = R.path([0], rows.map(row => ({ ...row, bytesUsed: row.kibUsed}))); + // const environment = R.path([0], rows); const { name: projectName } = await projectHelpers(sqlClientPool).getProjectByEnvironmentId(environment['environment']); userActivityLogger(`User updated environment storage on project '${projectName}'`, { diff --git a/services/api/src/typeDefs.js b/services/api/src/typeDefs.js index 835ec5d846..f17a51165d 100644 --- a/services/api/src/typeDefs.js +++ b/services/api/src/typeDefs.js @@ -950,13 +950,15 @@ const typeDefs = gql` id: Int environment: Environment persistentStorageClaim: String - bytesUsed: Float + bytesUsed: Float @deprecated(reason: "The value of this is kibibytes, use kibUsed instead") + kibUsed: Float updated: String } type EnvironmentStorageMonth { month: String - bytesUsed: Float + bytesUsed: Float @deprecated(reason: "The value of this is kibibytes, use kibUsed instead") + kibUsed: Float } type EnvironmentHoursMonth { @@ -1589,8 +1591,15 @@ const typeDefs = gql` input AddOrUpdateEnvironmentStorageInput { environment: Int! persistentStorageClaim: String! + """ + @deprecated(reason: "When bytesUsed is removed, kibUsed will become a required field") + """ bytesUsed: Int! """ + kibUsed is a float to allow for greater than 32-bit integer inputs + """ + kibUsed: Float + """ Date in format 'YYYY-MM-DD' """ updated: String diff --git a/services/workflows/internal/lagoonclient/schema.graphql b/services/workflows/internal/lagoonclient/schema.graphql index d28d5ce864..c951d05ce9 100644 --- a/services/workflows/internal/lagoonclient/schema.graphql +++ b/services/workflows/internal/lagoonclient/schema.graphql @@ -194,8 +194,14 @@ input AddOpenshiftInput { input AddOrUpdateEnvironmentStorageInput { environment: Int! persistentStorageClaim: String! + """ + @deprecated(reason: "When bytesUsed is removed, kibUsed will become a required field") + """ bytesUsed: Int! - + """ + kibUsed is a float to allow for greater than 32-bit integer inputs + """ + kibUsed: Float """ Date in format 'YYYY-MM-DD' """ @@ -811,13 +817,15 @@ type EnvironmentStorage { id: Int environment: Environment persistentStorageClaim: String - bytesUsed: Float + bytesUsed: Float @deprecated(reason: "The value of this is kibibytes, use kibUsed field instead") + kibUsed: Float updated: String } type EnvironmentStorageMonth { month: String - bytesUsed: Float + bytesUsed: Float @deprecated(reason: "The value of this is kibibytes, use kibUsed field instead") + kibUsed: Float } type EnvKeyValue {