diff --git a/node-packages/commons/src/tasks.ts b/node-packages/commons/src/tasks.ts index 9af665f082..95a2d93aee 100644 --- a/node-packages/commons/src/tasks.ts +++ b/node-packages/commons/src/tasks.ts @@ -798,104 +798,90 @@ export const createDeployTask = async function(deployData: any) { // production_environment: 'master', // environments: [ { name: 'develop', environment_type: 'development' }, [Object] ] } } - if (typeof project.activeSystemsDeploy === 'undefined') { - throw new UnknownActiveSystem( - `No active system for tasks 'deploy' in for project ${projectName}` + // we want to limit production environments, without making it configurable currently + var productionEnvironmentsLimit = 2; + + // we want to make sure we can deploy the `production` env, and also the env defined as standby + if ( + environments.project.productionEnvironment === branchName || + environments.project.standbyProductionEnvironment === branchName + ) { + // get a list of production environments + const prod_environments = environments.project.environments + .filter(e => e.environmentType === 'production') + .map(e => e.name); + logger.debug( + `projectName: ${projectName}, branchName: ${branchName}, existing environments are ${prod_environments}` ); - } - switch (project.activeSystemsDeploy) { - case 'lagoon_controllerBuildDeploy': - // we want to limit production environments, without making it configurable currently - var productionEnvironmentsLimit = 2; - - // we want to make sure we can deploy the `production` env, and also the env defined as standby - if ( - environments.project.productionEnvironment === branchName || - environments.project.standbyProductionEnvironment === branchName - ) { - // get a list of production environments - const prod_environments = environments.project.environments - .filter(e => e.environmentType === 'production') - .map(e => e.name); + if (prod_environments.length >= productionEnvironmentsLimit) { + if (prod_environments.find(i => i === branchName)) { logger.debug( - `projectName: ${projectName}, branchName: ${branchName}, existing environments are ${prod_environments}` + `projectName: ${projectName}, branchName: ${branchName}, environment already exists, no environment limits considered` ); - - if (prod_environments.length >= productionEnvironmentsLimit) { - if (prod_environments.find(i => i === branchName)) { - logger.debug( - `projectName: ${projectName}, branchName: ${branchName}, environment already exists, no environment limits considered` - ); - } else { - throw new EnvironmentLimit( - `'${branchName}' would exceed the configured limit of ${productionEnvironmentsLimit} production environments for project ${projectName}` - ); - } - } } else { - // get a list of non-production environments - const dev_environments = environments.project.environments - .filter(e => e.environmentType === 'development') - .map(e => e.name); - logger.debug( - `projectName: ${projectName}, branchName: ${branchName}, existing environments are ${dev_environments}` + throw new EnvironmentLimit( + `'${branchName}' would exceed the configured limit of ${productionEnvironmentsLimit} production environments for project ${projectName}` ); - - if ( - environments.project.developmentEnvironmentsLimit !== null && - dev_environments.length >= - environments.project.developmentEnvironmentsLimit - ) { - if (dev_environments.find(i => i === branchName)) { - logger.debug( - `projectName: ${projectName}, branchName: ${branchName}, environment already exists, no environment limits considered` - ); - } else { - throw new EnvironmentLimit( - `'${branchName}' would exceed the configured limit of ${environments.project.developmentEnvironmentsLimit} development environments for project ${projectName}` - ); - } - } } + } + } else { + // get a list of non-production environments + const dev_environments = environments.project.environments + .filter(e => e.environmentType === 'development') + .map(e => e.name); + logger.debug( + `projectName: ${projectName}, branchName: ${branchName}, existing environments are ${dev_environments}` + ); - if (type === 'branch') { - // use deployTargetBranches function to handle - let lagoonData = { - projectId: environments.project.id, - projectName, - branchName, - project, - deployData - } - try { - let result = deployTargetBranches(lagoonData) - return result - } catch (error) { - throw error - } - } else if (type === 'pullrequest') { - // use deployTargetPullrequest function to handle - let lagoonData = { - projectId: environments.project.id, - projectName, - branchName, - project, - pullrequestTitle, - deployData - } - try { - let result = deployTargetPullrequest(lagoonData) - return result - } catch (error) { - throw error - } + if ( + environments.project.developmentEnvironmentsLimit !== null && + dev_environments.length >= + environments.project.developmentEnvironmentsLimit + ) { + if (dev_environments.find(i => i === branchName)) { + logger.debug( + `projectName: ${projectName}, branchName: ${branchName}, environment already exists, no environment limits considered` + ); + } else { + throw new EnvironmentLimit( + `'${branchName}' would exceed the configured limit of ${environments.project.developmentEnvironmentsLimit} development environments for project ${projectName}` + ); } - break; - default: - throw new UnknownActiveSystem( - `Unknown active system '${project.activeSystemsDeploy}' for task 'deploy' in for project ${projectName}` - ); + } + } + + if (type === 'branch') { + // use deployTargetBranches function to handle + let lagoonData = { + projectId: environments.project.id, + projectName, + branchName, + project, + deployData + } + try { + let result = deployTargetBranches(lagoonData) + return result + } catch (error) { + throw error + } + } else if (type === 'pullrequest') { + // use deployTargetPullrequest function to handle + let lagoonData = { + projectId: environments.project.id, + projectName, + branchName, + project, + pullrequestTitle, + deployData + } + try { + let result = deployTargetPullrequest(lagoonData) + return result + } catch (error) { + throw error + } } } @@ -909,25 +895,12 @@ export const createPromoteTask = async function(promoteData: any) { const project = await getActiveSystemForProject(projectName, 'Promote'); - if (typeof project.activeSystemsPromote === 'undefined') { - throw new UnknownActiveSystem( - `No active system for tasks 'deploy' in for project ${projectName}` - ); - } - - switch (project.activeSystemsPromote) { - case 'lagoon_controllerBuildDeploy': - // use deployTargetPromote function to handle - let lagoonData = { - projectId: project.id, - promoteData - } - return deployTargetPromote(lagoonData) - default: - throw new UnknownActiveSystem( - `Unknown active system '${project.activeSystemsPromote}' for task 'deploy' in for project ${projectName}` - ); + // use deployTargetPromote function to handle + let lagoonData = { + projectId: project.id, + promoteData } + return deployTargetPromote(lagoonData) } export const createRemoveTask = async function(removeData: any) { @@ -958,114 +931,95 @@ export const createRemoveTask = async function(removeData: any) { } } - const project = await getActiveSystemForProject(projectName, 'Remove'); + if (type === 'branch') { + let environmentId = 0; + // Check to ensure the environment actually exists. + let foundEnvironment = false; + allEnvironments.project.environments.forEach(function( + environment, + index + ) { + if (environment.name === branch) { + foundEnvironment = true; + environmentId = environment.id; + } + }); - if (typeof project.activeSystemsRemove === 'undefined') { - throw new UnknownActiveSystem( - `No active system for tasks 'remove' in for project ${projectName}` + if (!foundEnvironment) { + logger.debug( + `projectName: ${projectName}, branchName: ${branch}, no environment found.` + ); + throw new NoNeedToRemoveBranch( + 'Branch environment does not exist, no need to remove anything.' + ); + } + // consume the deploytarget from the environment now + const result = await getOpenShiftInfoForEnvironment(environmentId); + const deployTarget = result.environment.openshift.name + logger.debug( + `projectName: ${projectName}, branchName: ${branchName}. Removing branch environment.` ); - } + // use the targetname as the routing key with the action + return sendToLagoonTasks(deployTarget+":remove", removeData); + } else if (type === 'pullrequest') { + // Work out the branch name from the PR number. + let branchName = 'pr-' + pullrequestNumber; + removeData.branchName = 'pr-' + pullrequestNumber; + + let environmentId = 0; + // Check to ensure the environment actually exists. + let foundEnvironment = false; + allEnvironments.project.environments.forEach(function( + environment, + index + ) { + if (environment.name === branchName) { + foundEnvironment = true; + environmentId = environment.id; + } + }); - switch (project.activeSystemsRemove) { - // removed `openshift` and `kubernetes` remove functionality, these services no longer exist in Lagoon - // handle removals using the controllers, send the message to our specific target cluster queue - case 'lagoon_controllerRemove': - if (type === 'branch') { - let environmentId = 0; - // Check to ensure the environment actually exists. - let foundEnvironment = false; - allEnvironments.project.environments.forEach(function( - environment, - index - ) { - if (environment.name === branch) { - foundEnvironment = true; - environmentId = environment.id; - } - }); - - if (!foundEnvironment) { - logger.debug( - `projectName: ${projectName}, branchName: ${branch}, no environment found.` - ); - throw new NoNeedToRemoveBranch( - 'Branch environment does not exist, no need to remove anything.' - ); - } - // consume the deploytarget from the environment now - const result = await getOpenShiftInfoForEnvironment(environmentId); - const deployTarget = result.environment.openshift.name - logger.debug( - `projectName: ${projectName}, branchName: ${branchName}. Removing branch environment.` - ); - // use the targetname as the routing key with the action - return sendToLagoonTasks(deployTarget+":remove", removeData); - } else if (type === 'pullrequest') { - // Work out the branch name from the PR number. - let branchName = 'pr-' + pullrequestNumber; - removeData.branchName = 'pr-' + pullrequestNumber; - - let environmentId = 0; - // Check to ensure the environment actually exists. - let foundEnvironment = false; - allEnvironments.project.environments.forEach(function( - environment, - index - ) { - if (environment.name === branchName) { - foundEnvironment = true; - environmentId = environment.id; - } - }); - - if (!foundEnvironment) { - logger.debug( - `projectName: ${projectName}, pullrequest: ${branchName}, no pullrequest found.` - ); - throw new NoNeedToRemoveBranch( - 'Pull Request environment does not exist, no need to remove anything.' - ); - } - // consume the deploytarget from the environment now - const result = await getOpenShiftInfoForEnvironment(environmentId); - const deployTarget = result.environment.openshift.name - logger.debug( - `projectName: ${projectName}, pullrequest: ${branchName}. Removing pullrequest environment.` - ); - return sendToLagoonTasks(deployTarget+":remove", removeData); - } else if (type === 'promote') { - let environmentId = 0; - // Check to ensure the environment actually exists. - let foundEnvironment = false; - allEnvironments.project.environments.forEach(function( - environment, - index - ) { - if (environment.name === branch) { - foundEnvironment = true; - environmentId = environment.id; - } - }); - - if (!foundEnvironment) { - logger.debug( - `projectName: ${projectName}, branchName: ${branch}, no environment found.` - ); - throw new NoNeedToRemoveBranch( - 'Branch environment does not exist, no need to remove anything.' - ); - } - // consume the deploytarget from the environment now - const result = await getOpenShiftInfoForEnvironment(environmentId); - const deployTarget = result.environment.openshift.name - return sendToLagoonTasks(deployTarget+":remove", removeData); + if (!foundEnvironment) { + logger.debug( + `projectName: ${projectName}, pullrequest: ${branchName}, no pullrequest found.` + ); + throw new NoNeedToRemoveBranch( + 'Pull Request environment does not exist, no need to remove anything.' + ); + } + // consume the deploytarget from the environment now + const result = await getOpenShiftInfoForEnvironment(environmentId); + const deployTarget = result.environment.openshift.name + logger.debug( + `projectName: ${projectName}, pullrequest: ${branchName}. Removing pullrequest environment.` + ); + return sendToLagoonTasks(deployTarget+":remove", removeData); + } else if (type === 'promote') { + let environmentId = 0; + // Check to ensure the environment actually exists. + let foundEnvironment = false; + allEnvironments.project.environments.forEach(function( + environment, + index + ) { + if (environment.name === branch) { + foundEnvironment = true; + environmentId = environment.id; } - break; + }); - default: - throw new UnknownActiveSystem( - `Unknown active system '${project.activeSystemsRemove}' for task 'remove' in for project ${projectName}` + if (!foundEnvironment) { + logger.debug( + `projectName: ${projectName}, branchName: ${branch}, no environment found.` ); + throw new NoNeedToRemoveBranch( + 'Branch environment does not exist, no need to remove anything.' + ); + } + // consume the deploytarget from the environment now + const result = await getOpenShiftInfoForEnvironment(environmentId); + const deployTarget = result.environment.openshift.name + return sendToLagoonTasks(deployTarget+":remove", removeData); } } @@ -1160,25 +1114,11 @@ export const createTaskTask = async function(taskData: any) { taskData.project.organization = organization } - if (typeof projectSystem.activeSystemsTask === 'undefined') { - throw new UnknownActiveSystem( - `No active system for 'task' for project ${project.name}` - ); - } - - switch (projectSystem.activeSystemsTask) { - case 'lagoon_controllerJob': - // since controllers queues are named, we have to send it to the right tasks queue - // do that here by querying which deploytarget the environment uses - const result = await getOpenShiftInfoForEnvironment(taskData.environment.id); - const deployTarget = result.environment.openshift.name - return sendToLagoonTasks(deployTarget+":jobs", taskData); - - default: - throw new UnknownActiveSystem( - `Unknown active system '${projectSystem.activeSystemsTask}' for 'task' for project ${project.name}` - ); - } + // since controllers queues are named, we have to send it to the right tasks queue + // do that here by querying which deploytarget the environment uses + const result = await getOpenShiftInfoForEnvironment(taskData.environment.id); + const deployTarget = result.environment.openshift.name + return sendToLagoonTasks(deployTarget+":jobs", taskData); } export const createMiscTask = async function(taskData: any) { @@ -1187,208 +1127,201 @@ export const createMiscTask = async function(taskData: any) { data: { project } } = taskData; - const data = await getActiveSystemForProject(project.name, 'Misc'); - - let updatedKey = key; - let taskId = ''; - switch (data.activeSystemsMisc) { - case 'lagoon_controllerMisc': - // handle any controller based misc tasks - updatedKey = `deploytarget:${key}`; - taskId = 'misc-kubernetes'; - // determine the deploy target (openshift/kubernetes) for the task to go to - // we get this from the environment - const result = await getOpenShiftInfoForEnvironment(taskData.data.environment.id); - const deployTarget = result.environment.openshift.name - // this is the json structure for sending a misc task to the controller - // there are some additional bits that can be adjusted, and these are done in the switch below on `updatedKey` - var miscTaskData: any = { - misc: {}, - key: updatedKey, - environment: { - name: taskData.data.environment.name, - openshiftProjectName: taskData.data.environment.openshiftProjectName - }, - project: { - name: taskData.data.project.name - }, - task: taskData.data.task, - advancedTask: {} + let updatedKey = `deploytarget:${key}`; + // determine the deploy target (openshift/kubernetes) for the task to go to + // we get this from the environment + const result = await getOpenShiftInfoForEnvironment(taskData.data.environment.id); + const deployTarget = result.environment.openshift.name + // this is the json structure for sending a misc task to the controller + // there are some additional bits that can be adjusted, and these are done in the switch below on `updatedKey` + var miscTaskData: any = { + misc: {}, + key: updatedKey, + environment: { + name: taskData.data.environment.name, + openshiftProjectName: taskData.data.environment.openshiftProjectName + }, + project: { + name: taskData.data.project.name + }, + task: taskData.data.task, + advancedTask: {} + } + switch (updatedKey) { + case 'deploytarget:restic:backup:restore': + // Handle setting up the configuration for a restic restoration task + const randRestoreId = Math.random().toString(36).substring(7); + const restoreName = `restore-${R.slice(0, 7, taskData.data.backup.backupId)}-${randRestoreId}`; + // Parse out the baasBucketName for any migrated projects + // check if the project is configured for a shared baas bucket + let [baasBucketName, shared] = await getBaasBucketName(result.environment.project, result.environment.openshift) + if (shared) { + // if it is a shared bucket, add the repo key to it too for restores + baasBucketName = `${baasBucketName}/baas-${makeSafe(taskData.data.project.name)}` + } + // Handle custom backup configurations + let lagoonBaasCustomBackupEndpoint = result.environment.project.envVariables.find(obj => { + return obj.name === "LAGOON_BAAS_CUSTOM_BACKUP_ENDPOINT" + }) + if (lagoonBaasCustomBackupEndpoint) { + lagoonBaasCustomBackupEndpoint = lagoonBaasCustomBackupEndpoint.value + } + let lagoonBaasCustomBackupBucket = result.environment.project.envVariables.find(obj => { + return obj.name === "LAGOON_BAAS_CUSTOM_BACKUP_BUCKET" + }) + if (lagoonBaasCustomBackupBucket) { + lagoonBaasCustomBackupBucket = lagoonBaasCustomBackupBucket.value + } + let lagoonBaasCustomBackupAccessKey = result.environment.project.envVariables.find(obj => { + return obj.name === "LAGOON_BAAS_CUSTOM_BACKUP_ACCESS_KEY" + }) + if (lagoonBaasCustomBackupAccessKey) { + lagoonBaasCustomBackupAccessKey = lagoonBaasCustomBackupAccessKey.value + } + let lagoonBaasCustomBackupSecretKey = result.environment.project.envVariables.find(obj => { + return obj.name === "LAGOON_BAAS_CUSTOM_BACKUP_SECRET_KEY" + }) + if (lagoonBaasCustomBackupSecretKey) { + lagoonBaasCustomBackupSecretKey = lagoonBaasCustomBackupSecretKey.value } - switch (updatedKey) { - case 'deploytarget:restic:backup:restore': - // Handle setting up the configuration for a restic restoration task - const randRestoreId = Math.random().toString(36).substring(7); - const restoreName = `restore-${R.slice(0, 7, taskData.data.backup.backupId)}-${randRestoreId}`; - // Parse out the baasBucketName for any migrated projects - // check if the project is configured for a shared baas bucket - let [baasBucketName, shared] = await getBaasBucketName(result.environment.project, result.environment.openshift) - if (shared) { - // if it is a shared bucket, add the repo key to it too for restores - baasBucketName = `${baasBucketName}/baas-${makeSafe(taskData.data.project.name)}` - } - // Handle custom backup configurations - let lagoonBaasCustomBackupEndpoint = result.environment.project.envVariables.find(obj => { - return obj.name === "LAGOON_BAAS_CUSTOM_BACKUP_ENDPOINT" - }) - if (lagoonBaasCustomBackupEndpoint) { - lagoonBaasCustomBackupEndpoint = lagoonBaasCustomBackupEndpoint.value - } - let lagoonBaasCustomBackupBucket = result.environment.project.envVariables.find(obj => { - return obj.name === "LAGOON_BAAS_CUSTOM_BACKUP_BUCKET" - }) - if (lagoonBaasCustomBackupBucket) { - lagoonBaasCustomBackupBucket = lagoonBaasCustomBackupBucket.value - } - let lagoonBaasCustomBackupAccessKey = result.environment.project.envVariables.find(obj => { - return obj.name === "LAGOON_BAAS_CUSTOM_BACKUP_ACCESS_KEY" - }) - if (lagoonBaasCustomBackupAccessKey) { - lagoonBaasCustomBackupAccessKey = lagoonBaasCustomBackupAccessKey.value - } - let lagoonBaasCustomBackupSecretKey = result.environment.project.envVariables.find(obj => { - return obj.name === "LAGOON_BAAS_CUSTOM_BACKUP_SECRET_KEY" - }) - if (lagoonBaasCustomBackupSecretKey) { - lagoonBaasCustomBackupSecretKey = lagoonBaasCustomBackupSecretKey.value - } - let backupS3Config = {} - if (lagoonBaasCustomBackupEndpoint && lagoonBaasCustomBackupBucket && lagoonBaasCustomBackupAccessKey && lagoonBaasCustomBackupSecretKey) { - backupS3Config = { - endpoint: lagoonBaasCustomBackupEndpoint, - bucket: lagoonBaasCustomBackupBucket, - accessKeyIDSecretRef: { - name: "lagoon-baas-custom-backup-credentials", - key: "access-key" - }, - secretAccessKeySecretRef: { - name: "lagoon-baas-custom-backup-credentials", - key: "secret-key" - } - } - } else { - backupS3Config = { - bucket: baasBucketName ? baasBucketName : `baas-${makeSafe(taskData.data.project.name)}` - } + let backupS3Config = {} + if (lagoonBaasCustomBackupEndpoint && lagoonBaasCustomBackupBucket && lagoonBaasCustomBackupAccessKey && lagoonBaasCustomBackupSecretKey) { + backupS3Config = { + endpoint: lagoonBaasCustomBackupEndpoint, + bucket: lagoonBaasCustomBackupBucket, + accessKeyIDSecretRef: { + name: "lagoon-baas-custom-backup-credentials", + key: "access-key" + }, + secretAccessKeySecretRef: { + name: "lagoon-baas-custom-backup-credentials", + key: "secret-key" } + } + } else { + backupS3Config = { + bucket: baasBucketName ? baasBucketName : `baas-${makeSafe(taskData.data.project.name)}` + } + } - // Handle custom restore configurations - let lagoonBaasCustomRestoreEndpoint = result.environment.project.envVariables.find(obj => { - return obj.name === "LAGOON_BAAS_CUSTOM_RESTORE_ENDPOINT" - }) - if (lagoonBaasCustomRestoreEndpoint) { - lagoonBaasCustomRestoreEndpoint = lagoonBaasCustomRestoreEndpoint.value - } - let lagoonBaasCustomRestoreBucket = result.environment.project.envVariables.find(obj => { - return obj.name === "LAGOON_BAAS_CUSTOM_RESTORE_BUCKET" - }) - if (lagoonBaasCustomRestoreBucket) { - lagoonBaasCustomRestoreBucket = lagoonBaasCustomRestoreBucket.value - } - let lagoonBaasCustomRestoreAccessKey = result.environment.project.envVariables.find(obj => { - return obj.name === "LAGOON_BAAS_CUSTOM_RESTORE_ACCESS_KEY" - }) - if (lagoonBaasCustomRestoreAccessKey) { - lagoonBaasCustomRestoreAccessKey = lagoonBaasCustomRestoreAccessKey.value - } - let lagoonBaasCustomRestoreSecretKey = result.environment.project.envVariables.find(obj => { - return obj.name === "LAGOON_BAAS_CUSTOM_RESTORE_SECRET_KEY" - }) - if (lagoonBaasCustomRestoreSecretKey) { - lagoonBaasCustomRestoreSecretKey = lagoonBaasCustomRestoreSecretKey.value - } + // Handle custom restore configurations + let lagoonBaasCustomRestoreEndpoint = result.environment.project.envVariables.find(obj => { + return obj.name === "LAGOON_BAAS_CUSTOM_RESTORE_ENDPOINT" + }) + if (lagoonBaasCustomRestoreEndpoint) { + lagoonBaasCustomRestoreEndpoint = lagoonBaasCustomRestoreEndpoint.value + } + let lagoonBaasCustomRestoreBucket = result.environment.project.envVariables.find(obj => { + return obj.name === "LAGOON_BAAS_CUSTOM_RESTORE_BUCKET" + }) + if (lagoonBaasCustomRestoreBucket) { + lagoonBaasCustomRestoreBucket = lagoonBaasCustomRestoreBucket.value + } + let lagoonBaasCustomRestoreAccessKey = result.environment.project.envVariables.find(obj => { + return obj.name === "LAGOON_BAAS_CUSTOM_RESTORE_ACCESS_KEY" + }) + if (lagoonBaasCustomRestoreAccessKey) { + lagoonBaasCustomRestoreAccessKey = lagoonBaasCustomRestoreAccessKey.value + } + let lagoonBaasCustomRestoreSecretKey = result.environment.project.envVariables.find(obj => { + return obj.name === "LAGOON_BAAS_CUSTOM_RESTORE_SECRET_KEY" + }) + if (lagoonBaasCustomRestoreSecretKey) { + lagoonBaasCustomRestoreSecretKey = lagoonBaasCustomRestoreSecretKey.value + } - let restoreS3Config = {} - if (lagoonBaasCustomRestoreEndpoint && lagoonBaasCustomRestoreBucket && lagoonBaasCustomRestoreAccessKey && lagoonBaasCustomRestoreSecretKey) { - restoreS3Config = { - endpoint: lagoonBaasCustomRestoreEndpoint, - bucket: lagoonBaasCustomRestoreBucket, - accessKeyIDSecretRef: { - name: "lagoon-baas-custom-restore-credentials", - key: "access-key" - }, - secretAccessKeySecretRef: { - name: "lagoon-baas-custom-restore-credentials", - key: "secret-key" - } - } + let restoreS3Config = {} + if (lagoonBaasCustomRestoreEndpoint && lagoonBaasCustomRestoreBucket && lagoonBaasCustomRestoreAccessKey && lagoonBaasCustomRestoreSecretKey) { + restoreS3Config = { + endpoint: lagoonBaasCustomRestoreEndpoint, + bucket: lagoonBaasCustomRestoreBucket, + accessKeyIDSecretRef: { + name: "lagoon-baas-custom-restore-credentials", + key: "access-key" + }, + secretAccessKeySecretRef: { + name: "lagoon-baas-custom-restore-credentials", + key: "secret-key" } + } + } - // generate the restore CRD - const restoreConf = restoreConfig(restoreName, taskData.data.backup.backupId, backupS3Config, restoreS3Config) - //logger.info(restoreConf) - // base64 encode it - const restoreBytes = new Buffer(JSON.stringify(restoreConf).replace(/\\n/g, "\n")).toString('base64') - miscTaskData.misc.miscResource = restoreBytes - break; - case 'deploytarget:task:activestandby': - // handle setting up the task configuration for running the active/standby switch - // this uses the `advanced task` system in the controllers - // generate out custom json payload to send to the advanced task - var jsonPayload: any = { - productionEnvironment: taskData.data.productionEnvironment.name, - standbyEnvironment: taskData.data.environment.name, - sourceNamespace: makeSafe(taskData.data.environment.openshiftProjectName), - destinationNamespace: makeSafe(taskData.data.productionEnvironment.openshiftProjectName) - } - // encode it - const jsonPayloadBytes = new Buffer(JSON.stringify(jsonPayload).replace(/\\n/g, "\n")).toString('base64') - // set the task data up - miscTaskData.advancedTask.JSONPayload = jsonPayloadBytes - // use this image to run the task - let taskImage = "" - // choose which task image to use - if (CI == "true") { - taskImage = "172.17.0.1:5000/lagoon/task-activestandby:latest" - } else if (overwriteActiveStandbyTaskImage) { - // allow to overwrite the image we use via OVERWRITE_ACTIVESTANDBY_TASK_IMAGE env variable - taskImage = overwriteActiveStandbyTaskImage - } else { - taskImage = `uselagoon/task-activestandby:${lagoonVersion}` - } - miscTaskData.advancedTask.runnerImage = taskImage - // miscTaskData.advancedTask.runnerImage = "shreddedbacon/runner:latest" - break; - case 'deploytarget:task:advanced': - // inject variables into advanced tasks the same way it is in builds and standard tasks - const [_, envVars, projectVars] = await getTaskProjectEnvironmentVariables( - taskData.data.project.name, - taskData.data.environment.id - ) - miscTaskData.project.variables = { - project: projectVars, - environment: envVars, - } - miscTaskData.advancedTask = taskData.data.advancedTask - break; - case 'deploytarget:task:cancel': - // task cancellation is just a standard unmodified message - miscTaskData.misc = taskData.data.task - break; - case 'deploytarget:build:cancel': - // build cancellation is just a standard unmodified message - miscTaskData.misc = taskData.data.build - break; - default: - miscTaskData.misc = taskData.data.build - break; + // generate the restore CRD + const restoreConf = restoreConfig(restoreName, taskData.data.backup.backupId, backupS3Config, restoreS3Config) + //logger.info(restoreConf) + // base64 encode it + const restoreBytes = new Buffer(JSON.stringify(restoreConf).replace(/\\n/g, "\n")).toString('base64') + miscTaskData.misc.miscResource = restoreBytes + break; + case 'deploytarget:task:activestandby': + // handle setting up the task configuration for running the active/standby switch + // this uses the `advanced task` system in the controllers + // generate out custom json payload to send to the advanced task + var jsonPayload: any = { + productionEnvironment: taskData.data.productionEnvironment.name, + standbyEnvironment: taskData.data.environment.name, + sourceNamespace: makeSafe(taskData.data.environment.openshiftProjectName), + destinationNamespace: makeSafe(taskData.data.productionEnvironment.openshiftProjectName) } - // send the task to the queue - if (project.organization != null) { - const curOrg = await getOrganizationById(project.organization); - const organization = { - name: curOrg.name, - id: curOrg.id, - } - miscTaskData.project.organization = organization + // encode it + const jsonPayloadBytes = new Buffer(JSON.stringify(jsonPayload).replace(/\\n/g, "\n")).toString('base64') + // set the task data up + miscTaskData.advancedTask.JSONPayload = jsonPayloadBytes + // use this image to run the task + let taskImage = "" + // choose which task image to use + if (CI == "true") { + taskImage = "172.17.0.1:5000/lagoon/task-activestandby:latest" + } else if (overwriteActiveStandbyTaskImage) { + // allow to overwrite the image we use via OVERWRITE_ACTIVESTANDBY_TASK_IMAGE env variable + taskImage = overwriteActiveStandbyTaskImage + } else { + taskImage = `uselagoon/task-activestandby:${lagoonVersion}` } - return sendToLagoonTasks(deployTarget+':misc', miscTaskData); + miscTaskData.advancedTask.runnerImage = taskImage + // miscTaskData.advancedTask.runnerImage = "shreddedbacon/runner:latest" + break; + case 'deploytarget:task:advanced': + // inject variables into advanced tasks the same way it is in builds and standard tasks + const [_, envVars, projectVars] = await getTaskProjectEnvironmentVariables( + taskData.data.project.name, + taskData.data.environment.id + ) + miscTaskData.project.variables = { + project: projectVars, + environment: envVars, + } + miscTaskData.advancedTask = taskData.data.advancedTask + break; + case 'deploytarget:task:cancel': + // task cancellation is just a standard unmodified message + miscTaskData.misc = taskData.data.task + break; + case 'deploytarget:build:cancel': + // build cancellation is just a standard unmodified message + miscTaskData.misc = taskData.data.build + break; + case 'deploytarget:unidle:environment': + // environment unidle request is just a standard message + break; + case 'deploytarget:idle:environment': + // environment idle request is just a standard message + break; default: + miscTaskData.misc = taskData.data.build break; } - - return sendToLagoonTasks(taskId, { ...taskData, key: updatedKey }); + // send the task to the queue + if (project.organization != null) { + const curOrg = await getOrganizationById(project.organization); + const organization = { + name: curOrg.name, + id: curOrg.id, + } + miscTaskData.project.organization = organization + } + return sendToLagoonTasks(deployTarget+':misc', miscTaskData); } export const consumeTasks = async function( diff --git a/services/actions-handler/go.mod b/services/actions-handler/go.mod index 779e5054a9..8435555195 100644 --- a/services/actions-handler/go.mod +++ b/services/actions-handler/go.mod @@ -4,14 +4,14 @@ go 1.21 require ( github.com/cheshir/go-mq/v2 v2.0.1 - github.com/uselagoon/machinery v0.0.9 + github.com/uselagoon/machinery v0.0.15-0.20231215023103-5cdfc5838e87 gopkg.in/matryer/try.v1 v1.0.0-20150601225556-312d2599e12e ) require ( github.com/NeowayLabs/wabbit v0.0.0-20210927194032-73ad61d1620e // indirect github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 // indirect - github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect + github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/uuid v1.3.0 // indirect github.com/guregu/null v4.0.0+incompatible // indirect diff --git a/services/actions-handler/go.sum b/services/actions-handler/go.sum index 4f43160325..df76fc3b0a 100644 --- a/services/actions-handler/go.sum +++ b/services/actions-handler/go.sum @@ -303,7 +303,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= @@ -407,6 +406,8 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -838,8 +839,8 @@ github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/uselagoon/machinery v0.0.9 h1:wj9CVSUtneh/ynt2by5UsaUoveFxkI7MY1c/EbR90p8= -github.com/uselagoon/machinery v0.0.9/go.mod h1:IXLxlkahEAEgpCmu9Xa/Wmjo6ja4Aoq7tf8G7VrileE= +github.com/uselagoon/machinery v0.0.15-0.20231215023103-5cdfc5838e87 h1:l6iDDMEy9pa0215arObnKpEXbaClAw7gtE8ACJRjfQ4= +github.com/uselagoon/machinery v0.0.15-0.20231215023103-5cdfc5838e87/go.mod h1:h/qeMWQR4Qqu33x+8AulNDeolEwvb/G+aIsn/jyUtwk= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= diff --git a/services/actions-handler/handler/controller_builds.go b/services/actions-handler/handler/controller_builds.go index 49c29cee60..4e53022a30 100644 --- a/services/actions-handler/handler/controller_builds.go +++ b/services/actions-handler/handler/controller_builds.go @@ -39,7 +39,7 @@ func (m *Messenger) handleBuild(ctx context.Context, messageQueue *mq.MessageQue // set up a lagoon client for use in the following process l := lclient.New(m.LagoonAPI.Endpoint, "actions-handler", &token, false) - deployment, err := lagoon.GetDeploymentByName(ctx, message.Namespace, message.Meta.BuildName, l) + deployment, err := lagoon.GetDeploymentByName(ctx, message.Meta.Project, message.Meta.Environment, message.Meta.BuildName, false, l) if err != nil { m.toLagoonLogs(messageQueue, map[string]interface{}{ "severity": "error", @@ -87,7 +87,7 @@ func (m *Messenger) handleBuild(ctx context.Context, messageQueue *mq.MessageQue "message": err.Error(), }) if m.EnableDebug { - log.Println(fmt.Sprintf("%sERROR: unable to get project - %v", prefix, err)) + log.Println(fmt.Sprintf("%sERROR: unable to get environment - %v", prefix, err)) } return err } diff --git a/services/actions-handler/handler/controller_idling.go b/services/actions-handler/handler/controller_idling.go new file mode 100644 index 0000000000..19cb958c2d --- /dev/null +++ b/services/actions-handler/handler/controller_idling.go @@ -0,0 +1,85 @@ +package handler + +import ( + "context" + "fmt" + "log" + "time" + + mq "github.com/cheshir/go-mq/v2" + "github.com/uselagoon/machinery/api/lagoon" + lclient "github.com/uselagoon/machinery/api/lagoon/client" + "github.com/uselagoon/machinery/api/schema" + "github.com/uselagoon/machinery/utils/jwt" +) + +func (m *Messenger) handleIdling(ctx context.Context, messageQueue *mq.MessageQueue, message *schema.LagoonMessage, messageID string) error { + prefix := fmt.Sprintf("(messageid:%s) %s: ", messageID, message.Namespace) + log.Println(fmt.Sprintf("%sreceived idling environment status update", prefix)) + // generate a lagoon token with a expiry of 60 seconds from now + token, err := jwt.GenerateAdminToken(m.LagoonAPI.TokenSigningKey, m.LagoonAPI.JWTAudience, m.LagoonAPI.JWTSubject, m.LagoonAPI.JWTIssuer, time.Now().Unix(), 60) + if err != nil { + // the token wasn't generated + if m.EnableDebug { + log.Println(fmt.Sprintf("ERROR: unable to generate token: %v", err)) + } + return nil + } + // set up a lagoon client for use in the following process + l := lclient.New(m.LagoonAPI.Endpoint, "actions-handler", &token, false) + var environmentID uint + // determine the environment id from the message + if message.Meta.ProjectID == nil && message.Meta.EnvironmentID == nil { + project, err := lagoon.GetMinimalProjectByName(ctx, message.Meta.Project, l) + if err != nil { + // send the log to the lagoon-logs exchange to be processed + m.toLagoonLogs(messageQueue, map[string]interface{}{ + "severity": "error", + "event": fmt.Sprintf("actions-handler:%s:failed", "updateEnvironment"), + "meta": project, + "message": err.Error(), + }) + if m.EnableDebug { + log.Println(fmt.Sprintf("%sERROR: unable to get project - %v", prefix, err)) + } + return err + } + environment, err := lagoon.GetEnvironmentByName(ctx, message.Meta.Environment, project.ID, l) + if err != nil { + // send the log to the lagoon-logs exchange to be processed + m.toLagoonLogs(messageQueue, map[string]interface{}{ + "severity": "error", + "event": fmt.Sprintf("actions-handler:%s:failed", "updateEnvironment"), + "meta": project, + "message": err.Error(), + }) + if m.EnableDebug { + log.Println(fmt.Sprintf("%sERROR: unable to get environment - %v", prefix, err)) + } + return err + } + environmentID = environment.ID + } else { + // pull the id from the message + environmentID = *message.Meta.EnvironmentID + } + updateEnvironmentPatch := schema.UpdateEnvironmentPatchInput{ + Idled: &message.Idled, + } + updateEnvironment, err := lagoon.UpdateEnvironment(ctx, environmentID, updateEnvironmentPatch, l) + if err != nil { + // send the log to the lagoon-logs exchange to be processed + m.toLagoonLogs(messageQueue, map[string]interface{}{ + "severity": "error", + "event": fmt.Sprintf("actions-handler:%s:failed", "updateDeployment"), + "meta": updateEnvironment, + "message": err.Error(), + }) + if m.EnableDebug { + log.Println(fmt.Sprintf("%sERROR: unable to update environment - %v", prefix, err)) + } + return err + } + log.Println(fmt.Sprintf("%supdated environment", prefix)) + return nil +} diff --git a/services/actions-handler/handler/handler.go b/services/actions-handler/handler/handler.go index 44bfca187f..b7809182af 100644 --- a/services/actions-handler/handler/handler.go +++ b/services/actions-handler/handler/handler.go @@ -142,6 +142,8 @@ func (m *Messenger) Consumer() { err = m.handleRemoval(ctx, messageQueue, logMsg, messageID) case "task": err = m.handleTask(ctx, messageQueue, logMsg, messageID) + case "idling": + err = m.handleIdling(ctx, messageQueue, logMsg, messageID) } // if there aren't any errors, then ack the message, an error indicates that there may have been an issue with the api handling the request // skipping this means the message will remain in the queue diff --git a/services/api/database/migrations/20231215000000_environment_idling_status.js b/services/api/database/migrations/20231215000000_environment_idling_status.js new file mode 100644 index 0000000000..c2f7ec300a --- /dev/null +++ b/services/api/database/migrations/20231215000000_environment_idling_status.js @@ -0,0 +1,21 @@ +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.up = async function(knex) { + return knex.schema + .alterTable('environment', (table) => { + table.boolean('idled').notNullable().defaultTo(0); + }) +}; + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.down = async function(knex) { + return knex.schema + .alterTable('environment', (table) => { + table.dropColumn('idled'); + }) +}; diff --git a/services/api/src/resolvers.js b/services/api/src/resolvers.js index 205eb1feb4..b55ba24733 100644 --- a/services/api/src/resolvers.js +++ b/services/api/src/resolvers.js @@ -124,6 +124,8 @@ const { userCanSshToEnvironment, getEnvironmentUrl, getEnvironmentsByKubernetes, + idleEnvironment, + unidleEnvironment, } = require('./resources/environment/resolvers'); const { @@ -714,7 +716,9 @@ const resolvers = { removeDeployTargetFromOrganization, updateEnvironmentDeployTarget, removeUserFromOrganizationGroups, - bulkImportProjectsAndGroupsToOrganization + bulkImportProjectsAndGroupsToOrganization, + idleEnvironment, + unidleEnvironment }, Subscription: { backupChanged: backupSubscriber, diff --git a/services/api/src/resources/environment/resolvers.ts b/services/api/src/resources/environment/resolvers.ts index a90946a938..735c7b275a 100644 --- a/services/api/src/resources/environment/resolvers.ts +++ b/services/api/src/resources/environment/resolvers.ts @@ -2,7 +2,10 @@ import * as R from 'ramda'; // @ts-ignore import { sendToLagoonLogs } from '@lagoon/commons/dist/logs/lagoon-logger'; // @ts-ignore -import { createRemoveTask } from '@lagoon/commons/dist/tasks'; +import { + createRemoveTask, + createMiscTask +} from '@lagoon/commons/dist/tasks'; import { ResolverFn } from '../'; import { logger } from '../../loggers/logger'; import { isPatchEmpty, query, knex } from '../../util/db'; @@ -668,7 +671,8 @@ export const updateEnvironment: ResolverFn = async ( route: input.patch.route, routes: input.patch.routes, autoIdle: input.patch.autoIdle, - created: input.patch.created + created: input.patch.created, + idled: input.patch.idled } }) ); @@ -693,7 +697,8 @@ export const updateEnvironment: ResolverFn = async ( route: input.patch.route, routes: input.patch.routes, autoIdle: input.patch.autoIdle, - created: input.patch.created + created: input.patch.created, + idled: input.patch.idled }, data: withK8s } @@ -808,3 +813,131 @@ export const userCanSshToEnvironment: ResolverFn = async ( return null; } }; + +export const idleEnvironment = async ( + root, + args, + { sqlClientPool, hasPermission, userActivityLogger } +) => { + const environment = await Helpers(sqlClientPool).getEnvironmentById(args.id); + + if (!environment) { + throw new Error( + 'Invalid environment ID' + ); + } + + await hasPermission('environment', 'view', { + project: environment.project + }); + + if (!environment.idled) { + const project = await projectHelpers(sqlClientPool).getProjectById( + environment.project + ); + + await hasPermission('deployment', 'cancel', { + project: project.id + }); + + const data = { + environment, + project, + disableAutomaticUnidling: args.disableAutomaticUnidling, + }; + + userActivityLogger(`User requested environment idling for '${environment.name}'`, { + project: '', + event: 'api:idleEnvironment', + payload: { + project: project.name, + environment: environment.name, + disableAutomaticUnidling: args.disableAutomaticUnidling, + } + }); + + try { + await createMiscTask({ key: 'idle:environment', data }); + return 'success'; + } catch (error) { + sendToLagoonLogs( + 'error', + '', + '', + 'api:idleEnvironment', + { environment: environment.id }, + `Environment idle attempt possibly failed, reason: ${error}` + ); + throw new Error( + error.message + ); + } + } else { + throw new Error( + `environment is already idled` + ); + } +}; + +export const unidleEnvironment = async ( + root, + args, + { sqlClientPool, hasPermission, userActivityLogger } +) => { + const environment = await Helpers(sqlClientPool).getEnvironmentById(args.id); + + if (!environment) { + throw new Error( + 'Invalid environment ID' + ); + } + + await hasPermission('environment', 'view', { + project: environment.project + }); + + if (environment.idled) { + const project = await projectHelpers(sqlClientPool).getProjectById( + environment.project + ); + + await hasPermission('deployment', 'cancel', { + project: project.id + }); + + const data = { + environment, + project, + }; + + userActivityLogger(`User requested environment unidle for '${environment.name}'`, { + project: '', + event: 'api:unidleEnvironment', + payload: { + project: project.name, + environment: environment.name + } + }); + + try { + await createMiscTask({ key: 'unidle:environment', data }); + return 'success'; + } catch (error) { + sendToLagoonLogs( + 'error', + '', + '', + 'api:unidleEnvironment', + { environment: environment.id }, + `Environment unidle attempt possibly failed, reason: ${error}` + ); + throw new Error( + error.message + ); + } + } else { + throw new Error( + `environment is already unidled` + ); + } +}; \ No newline at end of file diff --git a/services/api/src/typeDefs.js b/services/api/src/typeDefs.js index 12f577e019..c5dc5b408b 100644 --- a/services/api/src/typeDefs.js +++ b/services/api/src/typeDefs.js @@ -928,6 +928,10 @@ const typeDefs = gql` kubernetes: Kubernetes kubernetesNamespacePattern: String workflows: [Workflow] + """ + Is the environment currently idled + """ + idled: Boolean } type EnvironmentHitsMonth { @@ -2471,6 +2475,8 @@ const typeDefs = gql` This mutation performs a lot of actions, on big project and group imports, if it times out, subsequent runs will perform only the changes necessary """ bulkImportProjectsAndGroupsToOrganization(input: AddProjectToOrganizationInput, detachNotification: Boolean): ProjectGroupsToOrganization + idleEnvironment(id: Int!, disableAutomaticUnidling: Boolean): String + unidleEnvironment(id: Int!): String } type Subscription {