Change: deprecate environment storage bytesUsed for kibUsed

DEPRECATIONS.md

@@ -15,6 +15,8 @@ release link: https://github.com/uselagoon/lagoon/releases/tag/v2.18.0
 * The standard drupal based tasks that Lagoon ships with (drush ....) have been flagged as deprecated and should not be used anymore. These will need to be replaced with [custom tasks](https://docs.lagoon.sh/using-lagoon-advanced/custom-tasks/). Example replacement tasks will be provided prior to their removal.
 * This release introduces a deprecation of the `setEnvironmentServices` mutation to updated services for an environment, it is being replaced with `addOrUpdateEnvironmentService` and `deleteEnvironmentService`. This is becaues the type is being refactored to support additional information, and eventually additional functionality. For now, the actions-handler service will still support the older `setEnvironmentServices` for backwards compatability for a short period to allow older versions of `lagoon-remote` to still work, but a new version of `lagoon-remote` will be available that will no longer provides the payload that the actions-handler uses.
 * The value for `registry` which was previously required by the `lagoon-core` chart is no longer required. If you are using this, you will need to add it under the new `unauthenticatedRegistry` setting when installing `lagoon-remote` in the `lagoon-build-deploy` section of your values file. If you aren't using an actual registry and have the example `disabled-only-use-harbor-via-deploy-controller.invalid` value, then you do not need to do anything except you can now remove the `registry` setting from your core values file.
+* The environment storage return field named `bytesUsed` is deprecated. The actual value stored is `kibibytes`. A new return field called `kibUsed` exists and should be used, the returned data is the same and both fields are still returned. `bytesUsed` will be removed in a future release, make any adjustments now to use `kibUsed`. This will be a breaking change in a future release.
+* `addOrUpdateEnvironmentStorage` is deprecated, `addOrUpdateStorageOnEnvironment` is the replacement to use as it supports the updated input value for `kibUsed`. `addOrUpdateEnvironmentStorage` will be completely removed in a future release.
 
 ### Lagoon v2.17.0
 

local-dev/api-data-watcher-pusher/api-data/01-populate-api-data-lagoon-demo.gql

@@ -357,6 +357,24 @@ mutation PopulateApi {
     type
   }
 
+  UIProject1Environment1addStorage: addOrUpdateStorageOnEnvironment(input:{
+    environment: 3
+    persistentStorageClaim: "nginx"
+    kibUsed: 200000
+  }) {
+    id
+    kibUsed
+  }
+
+  UIProject1Environment1addStorage: addOrUpdateStorageOnEnvironment(input:{
+    environment: 3
+    persistentStorageClaim: "mariadb"
+    kibUsed: 200000
+  }) {
+    id
+    kibUsed
+  }
+
   UIProject1Environment1addFacts: addFacts(
     input: {
       facts: [

services/actions-handler/go.mod

@@ -4,7 +4,7 @@ go 1.21
 
 require (
 	github.com/cheshir/go-mq/v2 v2.0.1
-	github.com/uselagoon/machinery v0.0.17-0.20240108054822-78639cc0a1f3
+	github.com/uselagoon/machinery v0.0.17
 	gopkg.in/matryer/try.v1 v1.0.0-20150601225556-312d2599e12e
 )
 

services/actions-handler/go.sum

@@ -839,20 +839,8 @@ github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-github.com/uselagoon/machinery v0.0.17-0.20240108020517-e6622621374c h1:xEUsDgNpM3ZaD0S6dsxQ+j/8hrnMM1HLEhbDUxrqFHE=
-github.com/uselagoon/machinery v0.0.17-0.20240108020517-e6622621374c/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
-github.com/uselagoon/machinery v0.0.17-0.20240108024302-9784c2db7f1b h1:eF3O0RMETo+Bk3/I1SLjwkTvqbn/SkwCyTPWOFAl6ek=
-github.com/uselagoon/machinery v0.0.17-0.20240108024302-9784c2db7f1b/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
-github.com/uselagoon/machinery v0.0.17-0.20240108025245-bf15fc15ab08 h1:SRCSGYkw6G77d20f5aA0Q/TJYhczybgEU4g7SodVsNg=
-github.com/uselagoon/machinery v0.0.17-0.20240108025245-bf15fc15ab08/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
-github.com/uselagoon/machinery v0.0.17-0.20240108043534-c066518389be h1:cOBkrQa7DYDnIVZebJViIc9XEggFKYjfSKzqHC9VckE=
-github.com/uselagoon/machinery v0.0.17-0.20240108043534-c066518389be/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
-github.com/uselagoon/machinery v0.0.17-0.20240108043655-6cf0c0b42884 h1:33GmXqKEkpPzsspIse3xx/aN/Leh+Jdl9QcRrbnEktg=
-github.com/uselagoon/machinery v0.0.17-0.20240108043655-6cf0c0b42884/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
-github.com/uselagoon/machinery v0.0.17-0.20240108050446-30ff0a7df794 h1:2LP/ytk7sY6BrVY67PizVVYF6EnZKPzwKEOJg8JFY1I=
-github.com/uselagoon/machinery v0.0.17-0.20240108050446-30ff0a7df794/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
-github.com/uselagoon/machinery v0.0.17-0.20240108054822-78639cc0a1f3 h1:DYklzy44C1s1a1O6LqAi8RUpuqDzTzJTnW9IRQ8J91k=
-github.com/uselagoon/machinery v0.0.17-0.20240108054822-78639cc0a1f3/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
+github.com/uselagoon/machinery v0.0.17 h1:rykgkboGvSX+aMa6MPXtR22DJ67eTPC4WITwpVDuj+Y=
+github.com/uselagoon/machinery v0.0.17/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
 github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
 github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=

services/actions-handler/handler/action_storage.go

@@ -23,7 +23,8 @@ type Storage struct {
 type StorageClaim struct {
 	Environment          int    `json:"environment"`
 	PersisteStorageClaim string `json:"persistentStorageClaim"`
-	BytesUsed            int    `json:"bytesUsed"`
+	BytesUsed            uint64 `json:"bytesUsed"`
+	KiBUsed              uint64 `json:"kibUsed"`
 }
 
 func (m *Messenger) handleUpdateStorage(ctx context.Context, messageQueue *mq.MessageQueue, action *Action, messageID string) error {
@@ -35,32 +36,54 @@ func (m *Messenger) handleUpdateStorage(ctx context.Context, messageQueue *mq.Me
 	if err != nil {
 		// the token wasn't generated
 		if m.EnableDebug {
-			log.Println(fmt.Sprintf("%sERROR: unable to generate token: %v", prefix, err))
+			log.Printf("%sERROR: unable to generate token: %v", prefix, err)
 		}
 		return nil
 	}
 	// the action data can contain multiple storage claims, so iterate over them here
 	var errs []error
 	for _, sc := range storageClaims.Claims {
-		sci := schema.UpdateEnvironmentStorageInput{}
-		scdata, _ := json.Marshal(sc)
-		json.Unmarshal(scdata, &sci)
 		l := lclient.New(m.LagoonAPI.Endpoint, "actions-handler", &token, false)
-		environment, err := lagoon.UpdateStorage(ctx, &sci, l)
-		if err != nil {
+		var envID int
+		var hasErr error
+		// if this is a newer storage calculator with the `kibUsed` field, use the new mutation
+		if sc.KiBUsed != 0 {
+			scoei := schema.UpdateStorageOnEnvironmentInput{}
+			scdata, _ := json.Marshal(sc)
+			json.Unmarshal(scdata, &scoei)
+			environment, err := lagoon.UpdateStorageOnEnvironment(ctx, &scoei, l)
+			if err != nil {
+				hasErr = err
+			} else {
+				envID = environment.ID
+			}
+		} else {
+			// else use the old mutation
+			// @DEPRECATED to be removed in a future release
+			sci := schema.UpdateEnvironmentStorageInput{}
+			scdata, _ := json.Marshal(sc)
+			json.Unmarshal(scdata, &sci)
+			environment, err := lagoon.UpdateStorage(ctx, &sci, l)
+			if err != nil {
+				hasErr = err
+			} else {
+				envID = environment.ID
+			}
+		}
+		if hasErr != nil {
 			// send the log to the lagoon-logs exchange to be processed
 			m.toLagoonLogs(messageQueue, map[string]interface{}{
 				"severity": "error",
 				"event":    fmt.Sprintf("actions-handler:%s:error", action.EventType),
-				"meta":     sci,
-				"message":  err.Error(),
+				"meta":     sc,
+				"message":  hasErr.Error(),
 			})
 			if m.EnableDebug {
-				log.Println(fmt.Sprintf("%sERROR: unable to update storage for environment %v in the api: %v", prefix, sci.Environment, err))
+				log.Printf("%sERROR: unable to update storage for environment %v in the api: %v", prefix, sc.Environment, hasErr)
 			}
 			// if the error is in LagoonAPIErrorCheck, this should be retried
-			if LagoonAPIRetryErrorCheck(err) == nil {
-				errs = append(errs, err)
+			if LagoonAPIRetryErrorCheck(hasErr) == nil {
+				errs = append(errs, hasErr)
 			}
 			// try and update the next storage claim if there is one
 			continue
@@ -69,11 +92,11 @@ func (m *Messenger) handleUpdateStorage(ctx context.Context, messageQueue *mq.Me
 		m.toLagoonLogs(messageQueue, map[string]interface{}{
 			"severity": "info",
 			"event":    fmt.Sprintf("actions-handler:%s:updated", action.EventType),
-			"meta":     sci,
-			"message":  fmt.Sprintf("updated environment: %v, storage claim: %s, id: %v", sci.Environment, sci.PersisteStorageClaim, environment.ID),
+			"meta":     sc,
+			"message":  fmt.Sprintf("updated environment: %v, storage claim: %s, id: %v", sc.Environment, sc.PersisteStorageClaim, envID),
 		})
 		if m.EnableDebug {
-			log.Println(fmt.Sprintf("%supdated environment: %v, storage claim: %s, id: %v", prefix, sci.Environment, sci.PersisteStorageClaim, environment.ID))
+			log.Printf("%supdated environment: %v, storage claim: %s, id: %v", prefix, sc.Environment, sc.PersisteStorageClaim, envID)
 		}
 	}
 	if len(errs) > 0 {

services/api/database/migrations/20240223000000_bytesused_kibused.js

@@ -0,0 +1,21 @@
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.up = async function(knex) {
+    return knex.schema
+    .alterTable('environment_storage', function (table) {
+        table.renameColumn('bytes_used', 'kib_used');
+    })
+};
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.down = async function(knex) {
+    return knex.schema
+    .alterTable('environment_storage', function (table) {
+        table.renameColumn('kib_used', 'bytes_used');
+    })
+};

services/api/src/mocks.js

@@ -353,15 +353,15 @@ mocks.EnvironmentStorage = (parent, args = {}, context, info) => ({
   id: faker.random.number(),
   environment: args.hasOwnProperty('environment') ? { id: args.environment.id } : { id: mocks.Environment().id },
   persistentStorageClaim: '',
-  bytesUsed: faker.random.number({ precision: 0.001 }), // Float
+  kibUsed: faker.random.number({ precision: 0.001 }), // Float
   updated: mocks.Date(),
 });
 
 mocks.EnvironmentStorageMonth = () => {
   const date = new Date(mocks.Date());
   return {
     month: `${date.getFullYear()}-${date.getMonth() + 1}`,
-    bytesUsed: faker.random.number({ precision: 0.001 }),
+    kibUsed: faker.random.number({ precision: 0.001 }),
   };
 };
 

services/api/src/models/environment.ts

@@ -74,18 +74,18 @@ export const EnvironmentModel = (clients: { sqlClientPool: Pool }) => {
   };
 
   const environmentStorageMonthByEnvironmentId = async (eid, month) => {
-    const str = `
-      SELECT
-        SUM(bytes_used) as bytes_used, max(DATE_FORMAT(updated, '%Y-%m')) as month
-      FROM
-        environment_storage
-      WHERE
-        environment = :eid
-        AND YEAR(updated) = YEAR(STR_TO_DATE(:month, '%Y-%m'))
-        AND MONTH(updated) = MONTH(STR_TO_DATE(:month, '%Y-%m'))
-    `;
-
-    const rows = await query(sqlClientPool, str, { eid, month });
+    let q = knex('environment_storage')
+    .select(knex.raw('SUM(kib_used) as kib_used'))
+    .select(knex.raw('SUM(kib_used) as bytes_used')) // @DEPRECATE when `bytesUsed` is completely removed, this can be removed
+    .select(knex.raw(`max(DATE_FORMAT(updated, '%Y-%m')) as month`))
+    .where('environment', eid)
+    .andWhere(knex.raw(`YEAR(updated) = YEAR(STR_TO_DATE(?, '%Y-%m'))`, month))
+    .andWhere(knex.raw(`MONTH(updated) = MONTH(STR_TO_DATE(?, '%Y-%m'))`, month))
+
+    const rows = await query(sqlClientPool, q.toString());
+
+    rows.map(row => ({ ...row, bytesUsed: row.kibUsed})); // @DEPRECATE when `bytesUsed` is completely removed, this can be removed
+
     return rows[0];
   };
 

services/api/src/resources/environment/resolvers.ts

@@ -157,7 +157,9 @@ export const getEnvironmentStorageByEnvironmentId: ResolverFn = async (
 
   const rows = await query(sqlClientPool, Sql.selectEnvironmentStorageByEnvironmentId(eid))
 
-  return rows;
+  // @DEPRECATE when `bytesUsed` is completely removed, this can be reverted
+  return rows.map(row => ({ ...row, bytesUsed: row.kibUsed}));
+  // return rows;
 };
 
 export const getEnvironmentStorageMonthByEnvironmentId: ResolverFn = async (
@@ -439,11 +441,22 @@ export const addOrUpdateEnvironmentStorage: ResolverFn = async (
       : convertDateToMYSQLDateFormat(new Date().toISOString())
   };
 
+
+  // @DEPRECATE when `bytesUsed` is completely removed, this block can be removed
+  if (input.kibUsed) {
+    // remove the bytesUsed input if kilobytes is provided
+    delete input.bytesUsed
+  } else {
+    // else set kibUsed to the old required input, then remove the old input
+    input.kibUsed = input.bytesUsed
+    delete input.bytesUsed
+  }
+
   const createOrUpdateSql = knex('environment_storage')
     .insert(input)
     .onConflict('id')
     .merge({
-      bytesUsed: input.bytesUsed
+      kibUsed: input.kibUsed
     }).toString();
 
   const { insertId } = await query(
@@ -459,7 +472,9 @@ export const addOrUpdateEnvironmentStorage: ResolverFn = async (
       .toString()
   );
 
-  const environment = R.path([0], rows);
+  // @DEPRECATE when `bytesUsed` is completely removed, this can be reverted
+  const environment = R.path([0], rows.map(row => ({ ...row, bytesUsed: row.kibUsed})));
+  // const environment = R.path([0], rows);
   const { name: projectName } = await projectHelpers(sqlClientPool).getProjectByEnvironmentId(environment['environment']);
 
   userActivityLogger(`User updated environment storage on project '${projectName}'`, {

services/api/src/typeDefs.js

@@ -950,13 +950,15 @@ const typeDefs = gql`
     id: Int
     environment: Environment
     persistentStorageClaim: String
-    bytesUsed: Float
+    bytesUsed: Float @deprecated(reason: "The value of this is kibibytes, use kibUsed instead. This will be removed in a future release.")
+    kibUsed: Float
     updated: String
   }
 
   type EnvironmentStorageMonth {
     month: String
-    bytesUsed: Float
+    bytesUsed: Float @deprecated(reason: "The value of this is kibibytes, use kibUsed instead. This will be removed in a future release.")
+    kibUsed: Float
   }
 
   type EnvironmentHoursMonth {
@@ -1596,6 +1598,19 @@ const typeDefs = gql`
     updated: String
   }
 
+  input AddOrUpdateStorageOnEnvironmentInput {
+    environment: Int!
+    persistentStorageClaim: String!
+    """
+    kibUsed is a float to allow for greater than 32-bit integer inputs
+    """
+    kibUsed: Float!
+    """
+    Date in format 'YYYY-MM-DD'
+    """
+    updated: String
+  }
+
   input AddBackupInput {
     id: Int
     environment: Int!
@@ -2320,6 +2335,9 @@ const typeDefs = gql`
     """
     addOrUpdateEnvironmentStorage(
       input: AddOrUpdateEnvironmentStorageInput!
+    ): EnvironmentStorage  @deprecated(reason: "Use addOrUpdateStorageonEnvironment instead")
+    addOrUpdateStorageonEnvironment(
+      input: AddOrUpdateStorageOnEnvironmentInput!
     ): EnvironmentStorage
     addNotificationSlack(input: AddNotificationSlackInput!): NotificationSlack
     updateNotificationSlack(

services/workflows/internal/lagoonclient/schema.graphql

@@ -194,8 +194,14 @@ input AddOpenshiftInput {
 input AddOrUpdateEnvironmentStorageInput {
   environment: Int!
   persistentStorageClaim: String!
+  """
+  @deprecated(reason: "When bytesUsed is removed, kibUsed will become a required field")
+  """
   bytesUsed: Int!
-
+  """
+  kibUsed is a float to allow for greater than 32-bit integer inputs
+  """
+  kibUsed: Float
   """
   Date in format 'YYYY-MM-DD'
   """
@@ -811,13 +817,15 @@ type EnvironmentStorage {
   id: Int
   environment: Environment
   persistentStorageClaim: String
-  bytesUsed: Float
+  bytesUsed: Float @deprecated(reason: "The value of this is kibibytes, use kibUsed field instead")
+  kibUsed: Float
   updated: String
 }
 
 type EnvironmentStorageMonth {
   month: String
-  bytesUsed: Float
+  bytesUsed: Float @deprecated(reason: "The value of this is kibibytes, use kibUsed field instead")
+  kibUsed: Float
 }
 
 type EnvKeyValue {