deprecate: environment storage bytesused for kibused

DEPRECATIONS.md

@@ -15,6 +15,8 @@ release link: https://github.com/uselagoon/lagoon/releases/tag/v2.18.0
 * The standard drupal based tasks that Lagoon ships with (drush ....) have been flagged as deprecated and should not be used anymore. These will need to be replaced with [custom tasks](https://docs.lagoon.sh/using-lagoon-advanced/custom-tasks/). Example replacement tasks will be provided prior to their removal.
 * This release introduces a deprecation of the `setEnvironmentServices` mutation to updated services for an environment, it is being replaced with `addOrUpdateEnvironmentService` and `deleteEnvironmentService`. This is becaues the type is being refactored to support additional information, and eventually additional functionality. For now, the actions-handler service will still support the older `setEnvironmentServices` for backwards compatability for a short period to allow older versions of `lagoon-remote` to still work, but a new version of `lagoon-remote` will be available that will no longer provides the payload that the actions-handler uses.
 * The value for `registry` which was previously required by the `lagoon-core` chart is no longer required. If you are using this, you will need to add it under the new `unauthenticatedRegistry` setting when installing `lagoon-remote` in the `lagoon-build-deploy` section of your values file. If you aren't using an actual registry and have the example `disabled-only-use-harbor-via-deploy-controller.invalid` value, then you do not need to do anything except you can now remove the `registry` setting from your core values file.
+* The environment storage return field named `bytesUsed` is deprecated. The actual value stored is `kibibytes`. A new return field called `kibUsed` exists and should be used, the returned data is the same and both fields are still returned. `bytesUsed` will be removed in a future release, make any adjustments now to use `kibUsed`. This will be a breaking change in a future release.
+* When adding or updating environment storage using `addOrUpdateEnvironmentStorage`, the `bytesUsed` input is still a required field, `kibUsed` is optional. If `kibUsed` is optionally provided, the value of it will be preferred over `bytesUsed`. `bytesUsed` will be completely removed in a future release, `kibUsed` will be the required field. This will be a breaking change in a future release.
 
 ### Lagoon v2.17.0
 

local-dev/api-data-watcher-pusher/api-data/01-populate-api-data-lagoon-demo.gql

@@ -357,6 +357,28 @@ mutation PopulateApi {
     type
   }
 
+  UIProject1Environment1addStorage: addOrUpdateEnvironmentStorage(input:{
+    environment: 3
+    persistentStorageClaim: "nginx"
+    bytesUsed: 200000
+    kibUsed: 200000
+  }) {
+    id
+    bytesUsed
+    kibUsed
+  }
+
+  UIProject1Environment1addStorage: addOrUpdateEnvironmentStorage(input:{
+    environment: 3
+    persistentStorageClaim: "mariadb"
+    bytesUsed: 200000
+    kibUsed: 200000
+  }) {
+    id
+    bytesUsed
+    kibUsed
+  }
+
   UIProject1Environment1addFacts: addFacts(
     input: {
       facts: [

services/actions-handler/go.mod

@@ -4,7 +4,7 @@ go 1.21
 
 require (
 	github.com/cheshir/go-mq/v2 v2.0.1
-	github.com/uselagoon/machinery v0.0.17-0.20240108054822-78639cc0a1f3
+	github.com/uselagoon/machinery v0.0.17-0.20240227042958-7b4006aaf6cd
 	gopkg.in/matryer/try.v1 v1.0.0-20150601225556-312d2599e12e
 )
 

services/actions-handler/go.sum

@@ -853,6 +853,16 @@ github.com/uselagoon/machinery v0.0.17-0.20240108050446-30ff0a7df794 h1:2LP/ytk7
 github.com/uselagoon/machinery v0.0.17-0.20240108050446-30ff0a7df794/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
 github.com/uselagoon/machinery v0.0.17-0.20240108054822-78639cc0a1f3 h1:DYklzy44C1s1a1O6LqAi8RUpuqDzTzJTnW9IRQ8J91k=
 github.com/uselagoon/machinery v0.0.17-0.20240108054822-78639cc0a1f3/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
+github.com/uselagoon/machinery v0.0.17-0.20240222230758-208090a0bc18 h1:hIX+Fx5Or5nQ1pNTvX7QBIhmDNq9QCcVATCZ+RxSgwI=
+github.com/uselagoon/machinery v0.0.17-0.20240222230758-208090a0bc18/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
+github.com/uselagoon/machinery v0.0.17-0.20240222231146-70b5b91a3632 h1:k+VKnVqpH3LB574QFoYS5oiiftspkNlIwxurqnsWPpM=
+github.com/uselagoon/machinery v0.0.17-0.20240222231146-70b5b91a3632/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
+github.com/uselagoon/machinery v0.0.17-0.20240223003225-61f03325cde5 h1:W24cLoV/iuwmbpBm4HGbc6M79vS2AM06aIeffaIBrNs=
+github.com/uselagoon/machinery v0.0.17-0.20240223003225-61f03325cde5/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
+github.com/uselagoon/machinery v0.0.17-0.20240223004536-adb1b1a6dd8c h1:lCQf1N54gLFFKIw8x770BZCP+vgf+kEauNwn8i80kio=
+github.com/uselagoon/machinery v0.0.17-0.20240223004536-adb1b1a6dd8c/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
+github.com/uselagoon/machinery v0.0.17-0.20240227042958-7b4006aaf6cd h1:YN8DtIXYqPL88OYM4NRTeAiKo+WJr1K6UraLEdJYGrQ=
+github.com/uselagoon/machinery v0.0.17-0.20240227042958-7b4006aaf6cd/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
 github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
 github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=

services/actions-handler/handler/action_storage.go

@@ -23,7 +23,8 @@ type Storage struct {
 type StorageClaim struct {
 	Environment          int    `json:"environment"`
 	PersisteStorageClaim string `json:"persistentStorageClaim"`
-	BytesUsed            int    `json:"bytesUsed"`
+	BytesUsed            uint64 `json:"bytesUsed"`
+	KiBUsed              uint64 `json:"kibUsed"`
 }
 
 func (m *Messenger) handleUpdateStorage(ctx context.Context, messageQueue *mq.MessageQueue, action *Action, messageID string) error {
@@ -43,6 +44,13 @@ func (m *Messenger) handleUpdateStorage(ctx context.Context, messageQueue *mq.Me
 	var errs []error
 	for _, sc := range storageClaims.Claims {
 		sci := schema.UpdateEnvironmentStorageInput{}
+
+		// @DEPRECATE when `bytesUsed` is completely removed, this can be removed
+		// it just ensures that the `bytesUsed` value is always populated in case only `kibUsed` is provided from the storage-calculator
+		if sc.KiBUsed != 0 && sc.BytesUsed == 0 {
+			sc.BytesUsed = sc.KiBUsed
+		}
+
 		scdata, _ := json.Marshal(sc)
 		json.Unmarshal(scdata, &sci)
 		l := lclient.New(m.LagoonAPI.Endpoint, "actions-handler", &token, false)

services/api/database/migrations/20240223000000_bytesused_kibused.js

@@ -0,0 +1,21 @@
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.up = async function(knex) {
+    return knex.schema
+    .alterTable('environment_storage', function (table) {
+        table.renameColumn('bytes_used', 'kib_used');
+    })
+};
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.down = async function(knex) {
+    return knex.schema
+    .alterTable('environment_storage', function (table) {
+        table.renameColumn('kib_used', 'bytes_used');
+    })
+};

services/api/src/mocks.js

@@ -353,15 +353,15 @@ mocks.EnvironmentStorage = (parent, args = {}, context, info) => ({
   id: faker.random.number(),
   environment: args.hasOwnProperty('environment') ? { id: args.environment.id } : { id: mocks.Environment().id },
   persistentStorageClaim: '',
-  bytesUsed: faker.random.number({ precision: 0.001 }), // Float
+  kibUsed: faker.random.number({ precision: 0.001 }), // Float
   updated: mocks.Date(),
 });
 
 mocks.EnvironmentStorageMonth = () => {
   const date = new Date(mocks.Date());
   return {
     month: `${date.getFullYear()}-${date.getMonth() + 1}`,
-    bytesUsed: faker.random.number({ precision: 0.001 }),
+    kibUsed: faker.random.number({ precision: 0.001 }),
   };
 };
 

services/api/src/models/environment.ts

@@ -74,18 +74,18 @@ export const EnvironmentModel = (clients: { sqlClientPool: Pool }) => {
   };
 
   const environmentStorageMonthByEnvironmentId = async (eid, month) => {
-    const str = `
-      SELECT
-        SUM(bytes_used) as bytes_used, max(DATE_FORMAT(updated, '%Y-%m')) as month
-      FROM
-        environment_storage
-      WHERE
-        environment = :eid
-        AND YEAR(updated) = YEAR(STR_TO_DATE(:month, '%Y-%m'))
-        AND MONTH(updated) = MONTH(STR_TO_DATE(:month, '%Y-%m'))
-    `;
-
-    const rows = await query(sqlClientPool, str, { eid, month });
+    let q = knex('environment_storage')
+    .select(knex.raw('SUM(kib_used) as kib_used'))
+    .select(knex.raw('SUM(kib_used) as bytes_used')) // @DEPRECATE when `bytesUsed` is completely removed, this can be removed
+    .select(knex.raw(`max(DATE_FORMAT(updated, '%Y-%m')) as month`))
+    .where('environment', eid)
+    .andWhere(knex.raw(`YEAR(updated) = YEAR(STR_TO_DATE(?, '%Y-%m'))`, month))
+    .andWhere(knex.raw(`MONTH(updated) = MONTH(STR_TO_DATE(?, '%Y-%m'))`, month))
+
+    const rows = await query(sqlClientPool, q.toString());
+
+    rows.map(row => ({ ...row, bytesUsed: row.kibUsed})); // @DEPRECATE when `bytesUsed` is completely removed, this can be removed
+
     return rows[0];
   };
 

services/api/src/resources/environment/resolvers.ts

@@ -157,7 +157,9 @@ export const getEnvironmentStorageByEnvironmentId: ResolverFn = async (
 
   const rows = await query(sqlClientPool, Sql.selectEnvironmentStorageByEnvironmentId(eid))
 
-  return rows;
+  // @DEPRECATE when `bytesUsed` is completely removed, this can be reverted
+  return rows.map(row => ({ ...row, bytesUsed: row.kibUsed}));
+  // return rows;
 };
 
 export const getEnvironmentStorageMonthByEnvironmentId: ResolverFn = async (
@@ -439,11 +441,22 @@ export const addOrUpdateEnvironmentStorage: ResolverFn = async (
       : convertDateToMYSQLDateFormat(new Date().toISOString())
   };
 
+
+  // @DEPRECATE when `bytesUsed` is completely removed, this block can be removed
+  if (input.kibUsed) {
+    // remove the bytesUsed input if kilobytes is provided
+    delete input.bytesUsed
+  } else {
+    // else set kibUsed to the old required input, then remove the old input
+    input.kibUsed = input.bytesUsed
+    delete input.bytesUsed
+  }
+
   const createOrUpdateSql = knex('environment_storage')
     .insert(input)
     .onConflict('id')
     .merge({
-      bytesUsed: input.bytesUsed
+      kibUsed: input.kibUsed
     }).toString();
 
   const { insertId } = await query(
@@ -459,7 +472,9 @@ export const addOrUpdateEnvironmentStorage: ResolverFn = async (
       .toString()
   );
 
-  const environment = R.path([0], rows);
+  // @DEPRECATE when `bytesUsed` is completely removed, this can be reverted
+  const environment = R.path([0], rows.map(row => ({ ...row, bytesUsed: row.kibUsed})));
+  // const environment = R.path([0], rows);
   const { name: projectName } = await projectHelpers(sqlClientPool).getProjectByEnvironmentId(environment['environment']);
 
   userActivityLogger(`User updated environment storage on project '${projectName}'`, {

services/api/src/typeDefs.js

@@ -950,13 +950,15 @@ const typeDefs = gql`
     id: Int
     environment: Environment
     persistentStorageClaim: String
-    bytesUsed: Float
+    bytesUsed: Float @deprecated(reason: "The value of this is kibibytes, use kibUsed instead")
+    kibUsed: Float
     updated: String
   }
 
   type EnvironmentStorageMonth {
     month: String
-    bytesUsed: Float
+    bytesUsed: Float @deprecated(reason: "The value of this is kibibytes, use kibUsed instead")
+    kibUsed: Float
   }
 
   type EnvironmentHoursMonth {
@@ -1589,8 +1591,15 @@ const typeDefs = gql`
   input AddOrUpdateEnvironmentStorageInput {
     environment: Int!
     persistentStorageClaim: String!
+    """
+    @deprecated(reason: "When bytesUsed is removed, kibUsed will become a required field")
+    """
     bytesUsed: Int!
     """
+    kibUsed is a float to allow for greater than 32-bit integer inputs
+    """
+    kibUsed: Float
+    """
     Date in format 'YYYY-MM-DD'
     """
     updated: String

services/workflows/internal/lagoonclient/schema.graphql

@@ -194,8 +194,14 @@ input AddOpenshiftInput {
 input AddOrUpdateEnvironmentStorageInput {
   environment: Int!
   persistentStorageClaim: String!
+  """
+  @deprecated(reason: "When bytesUsed is removed, kibUsed will become a required field")
+  """
   bytesUsed: Int!
-
+  """
+  kibUsed is a float to allow for greater than 32-bit integer inputs
+  """
+  kibUsed: Float
   """
   Date in format 'YYYY-MM-DD'
   """
@@ -811,13 +817,15 @@ type EnvironmentStorage {
   id: Int
   environment: Environment
   persistentStorageClaim: String
-  bytesUsed: Float
+  bytesUsed: Float @deprecated(reason: "The value of this is kibibytes, use kibUsed field instead")
+  kibUsed: Float
   updated: String
 }
 
 type EnvironmentStorageMonth {
   month: String
-  bytesUsed: Float
+  bytesUsed: Float @deprecated(reason: "The value of this is kibibytes, use kibUsed field instead")
+  kibUsed: Float
 }
 
 type EnvKeyValue {