From 9ba26a949bddc0b5c970a07a580d49f100d1803a Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Tue, 12 Mar 2024 21:49:23 -0500 Subject: [PATCH 1/2] Deprecate activeSystems* --- DEPRECATIONS.md | 1 + docs/interacting/create-project.gql | 2 - docs/interacting/graphql-queries.md | 2 - .../using-harbor/harbor-settings/README.md | 2 +- node-packages/commons/src/api.ts | 264 ++--- node-packages/commons/src/deploy-tasks.ts | 99 +- node-packages/commons/src/tasks.ts | 924 +++++++++--------- node-packages/commons/src/types.ts | 3 + .../20240312010000_remove_activesystems.js | 29 + services/api/src/mocks.js | 4 - .../api/src/resources/project/resolvers.ts | 15 - services/api/src/resources/project/sql.ts | 10 - services/api/src/typeDefs.js | 30 +- .../src/handlers/bitbucketBranchDeleted.ts | 2 - .../handlers/bitbucketPullRequestClosed.ts | 2 - .../handlers/bitbucketPullRequestUpdated.ts | 2 - .../src/handlers/bitbucketPush.ts | 2 - .../src/handlers/giteaBranchDeleted.ts | 2 - .../src/handlers/giteaPullRequestClosed.ts | 2 - .../src/handlers/giteaPullRequestOpened.ts | 2 - .../handlers/giteaPullRequestSynchronize.ts | 2 - .../webhooks2tasks/src/handlers/giteaPush.ts | 2 - .../src/handlers/githubBranchDeleted.ts | 2 - .../src/handlers/githubPullRequestClosed.ts | 2 - .../src/handlers/githubPullRequestOpened.ts | 2 - .../handlers/githubPullRequestSynchronize.ts | 2 - .../webhooks2tasks/src/handlers/githubPush.ts | 2 - .../src/handlers/gitlabBranchDeleted.ts | 2 - .../src/handlers/gitlabPullRequestClosed.ts | 2 - .../src/handlers/gitlabPullRequestOpened.ts | 2 - .../src/handlers/gitlabPullRequestUpdated.ts | 2 - .../webhooks2tasks/src/handlers/gitlabPush.ts | 2 - services/webhooks2tasks/src/types.ts | 88 +- .../webhooks2tasks/src/webhooks/projects.ts | 28 +- 34 files changed, 723 insertions(+), 816 deletions(-) create mode 100644 node-packages/commons/src/types.ts create mode 100644 services/api/database/migrations/20240312010000_remove_activesystems.js diff --git a/DEPRECATIONS.md b/DEPRECATIONS.md index 4108121a62..7be2984390 100644 --- a/DEPRECATIONS.md +++ b/DEPRECATIONS.md @@ -17,6 +17,7 @@ release link: https://github.com/uselagoon/lagoon/releases/tag/v2.18.0 * The value for `registry` which was previously required by the `lagoon-core` chart is no longer required. If you are using this, you will need to add it under the new `unauthenticatedRegistry` setting when installing `lagoon-remote` in the `lagoon-build-deploy` section of your values file. If you aren't using an actual registry and have the example `disabled-only-use-harbor-via-deploy-controller.invalid` value, then you do not need to do anything except you can now remove the `registry` setting from your core values file. * The environment storage return field named `bytesUsed` is deprecated. The actual value stored is `kibibytes`. A new return field called `kibUsed` exists and should be used, the returned data is the same and both fields are still returned. `bytesUsed` will be removed in a future release, make any adjustments now to use `kibUsed`. This will be a breaking change in a future release. * `addOrUpdateEnvironmentStorage` is deprecated, `addOrUpdateStorageOnEnvironment` is the replacement to use as it supports the updated input value for `kibUsed`. `addOrUpdateEnvironmentStorage` will be completely removed in a future release. +* The ability to configure projects to use different subsystems for managing deployments/tasks has been removed. Setting any values for the `activeSystems*` fields on the project will have no effect and the fields will be removed in a future release. ### Lagoon v2.17.0 diff --git a/docs/interacting/create-project.gql b/docs/interacting/create-project.gql index c855b449b7..223a15af42 100644 --- a/docs/interacting/create-project.gql +++ b/docs/interacting/create-project.gql @@ -50,8 +50,6 @@ mutation { id } gitUrl - activeSystemsDeploy - activeSystemsRemove branches pullrequests } diff --git a/docs/interacting/graphql-queries.md b/docs/interacting/graphql-queries.md index cec2d2bd1e..11b279dc80 100644 --- a/docs/interacting/graphql-queries.md +++ b/docs/interacting/graphql-queries.md @@ -298,8 +298,6 @@ mutation { id } gitUrl - activeSystemsDeploy - activeSystemsRemove branches pullrequests } diff --git a/docs/using-lagoon-advanced/using-harbor/harbor-settings/README.md b/docs/using-lagoon-advanced/using-harbor/harbor-settings/README.md index f95848cd2f..1321778aa4 100644 --- a/docs/using-lagoon-advanced/using-harbor/harbor-settings/README.md +++ b/docs/using-lagoon-advanced/using-harbor/harbor-settings/README.md @@ -1,6 +1,6 @@ # Running Harbor Locally -Lagoon supports running Harbor locally, and it is automatically used for hosting all Kubernetes-based builds (any time the project's `activeSystemsDeploy` value is set to `lagoon_kubernetesBuildDeploy`). When Harbor is ran locally, it makes use of MinIO as a storage backend, which is an AWS S3 compatible local storage solution. +Lagoon supports running Harbor locally, where it makes use of MinIO as a storage backend, which is an AWS S3 compatible local storage solution. ## Settings diff --git a/node-packages/commons/src/api.ts b/node-packages/commons/src/api.ts index 98e5b7c495..27cbbaad01 100644 --- a/node-packages/commons/src/api.ts +++ b/node-packages/commons/src/api.ts @@ -6,38 +6,70 @@ import { logger } from './logs/local-logger'; import { envHasConfig, getConfigFromEnv } from './util/config'; export interface Project { - slack: any; - name: string; - openshift: DeployTarget; + autoIdle: number; + availability: ProjectAvailability; + branches: string; + buildImage: string; + created: string; deploymentsDisabled: number; - sharedBaasBucket?: boolean; - routerPattern?: string; - envVariables?: any; - gitUrl?: string; - subfolder?: string; - activesystemsdeploy?: string; - activesystemsremove?: string; - branches?: string; - productionenvironment?: string; - autoidle?: number; - storagecalc?: number; - pullrequests?: string; - openshiftprojectpattern?: string; - productionRoutes?: string; - standbyRoutes?: string; - productionEnvironment?: string; - standbyProductionEnvironment?: string; - organization?: number; + // deployTargetConfigs: ; + developmentBuildPriority: number; + developmentEnvironmentsLimit: number; + // environments: ; + envVariables: EnvKeyValue[]; + factsUI: number; + gitUrl: string; + // groups: ; + id: number; + kubernetes: Kubernetes; + kubernetesNamespacePattern: string; + // metadata: ; + name: string; + // notifications: ; + openshift: Kubernetes; + openshiftProjectName: string; + openshiftProjectPattern: string; + organization: number; + privateKey: string; + problemsUI: string; + productionAlias: string; + productionBuildPriority: number; + productionEnvironment: string; + productionRoutes: string; + publicKey: string; + pullrequests: string; + routerPattern: string; + sharedBaasBucket: boolean; + standbyAlias: string; + standbyProductionEnvironment: string; + standbyRoutes: string; + storageCalc: number; + subfolder: string; } -export interface DeployTarget { +export interface Kubernetes { + buildImage: string; + cloudProvider: string; + cloudRegion: string; + consoleUrl: string; + created: string; + disabled: boolean; + friendlyName: string; + id: number; + monitoringConfig: any; name: string; - sharedBaasBucketName?: string; - routerPattern?: string; - disabled?: boolean; - id?: number - buildImage?: string - monitoringConfig?: any + routerPattern: string; + sharedBaasBucketName: string; + sshHost: string; + sshPort: string; + token: string; +} + +export interface EnvKeyValue { + id: number; + name: string; + scope: EnvVariableScope; + value: string; } interface GroupPatch { @@ -57,8 +89,6 @@ interface ProjectPatch { gitUrl?: string; subfolder?: string; routerPattern?: string; - activesystemsdeploy?: string; - activesystemsremove?: string; branches?: string; productionenvironment?: string; autoidle?: number; @@ -103,6 +133,20 @@ enum EnvType { DEVELOPMENT = 'development' } +enum ProjectAvailability { + STANDARD = 'STANDARD', + HIGH = 'HIGH', + POLYSITE = 'POLYSITE' +} + +export enum EnvVariableScope { + BUILD = 'build', + RUNTIME = 'runtime', + GLOBAL = 'global', + CONTAINER_REGISTRY = 'container_registry', + INTERNAL_CONTAINER_REGISTRY = 'internal_container_registry' +} + let transportOptions: { headers: { Authorization?: string @@ -116,8 +160,9 @@ let transportOptions: { const transport = new Transport(`${getConfigFromEnv('API_HOST', 'http://api:3000')}/graphql`, {transportOptions}); export const graphqlapi = new Lokka({ transport }); +export const lagoonApi = graphqlapi; -class ProjectNotFound extends Error { +export class ProjectNotFound extends Error { constructor(message) { super(message); this.name = 'ProjectNotFound'; @@ -138,13 +183,6 @@ class EnvironmentNotFound extends Error { } } -class NoActiveSystemsDefined extends Error { - constructor(message) { - super(message); - this.name = 'NoActiveSystemsDefined'; - } -} - const sshKeyFragment = graphqlapi.createFragment(` fragment on SshKey { id @@ -613,24 +651,6 @@ export const deleteProject = (name: string): Promise => { name } ); -export async function getProjectsByGitUrl(gitUrl: string): Promise { - const result = await graphqlapi.query(` - { - allProjects(gitUrl: "${gitUrl}") { - name - productionEnvironment - deploymentsDisabled - } - } - `); - - if (!result || !result.allProjects || !result.allProjects.length) { - throw new ProjectNotFound(`Cannot find project for git repo ${gitUrl}`); - } - - return result.allProjects; -} - export async function getProjectByName(project: string): Promise { const result = await graphqlapi.query(` { @@ -817,54 +837,6 @@ export async function getEmailInfoForProject( return result.project.emails; } -interface GetActiveSystemForProjectResult { - id: number; - branches: string; - pullrequests: string; - organization: number; - activeSystemsDeploy: string; - activeSystemsPromote: string; - activeSystemsRemove: string; - activeSystemsTask: string; - activeSystemsMisc: string; -} - -export async function getActiveSystemForProject( - project: string, - task: 'Deploy' | 'Promote' | 'Remove' | 'Task' | 'Misc' -): Promise { - const field = `activeSystems${task}`; - const result = await graphqlapi.query(` - { - project:projectByName(name: "${project}"){ - id - activeSystemsDeploy - activeSystemsPromote - activeSystemsRemove - activeSystemsTask - activeSystemsMisc - branches - pullrequests - organization - } - } - `); - - if (!result || !result.project) { - throw new ProjectNotFound( - `Cannot find active-systems information for project ${project}` - ); - } - - if (!result.project[field]) { - throw new NoActiveSystemsDefined( - `Cannot find active system for task ${task} in project ${project}` - ); - } - - return result.project; -} - export async function getEnvironmentByName( name: string, projectId: number, @@ -1118,40 +1090,71 @@ export async function deleteEnvironment( ); } -export const getOpenShiftInfoForProject = (project: string): Promise => +interface GetOpenshiftInfoForProjectResult { + project: Pick< + Project, + | 'autoIdle' + | 'availability' + | 'branches' + | 'buildImage' + | 'developmentBuildPriority' + | 'gitUrl' + | 'id' + | 'openshiftProjectPattern' + | 'organization' + | 'privateKey' + | 'productionAlias' + | 'productionBuildPriority' + | 'productionEnvironment' + | 'productionRoutes' + | 'pullrequests' + | 'routerPattern' + | 'sharedBaasBucket' + | 'standbyAlias' + | 'standbyProductionEnvironment' + | 'standbyRoutes' + | 'storageCalc' + | 'subfolder' + > & { + openshift: Pick; + envVariables: Pick[]; + }; +} + +export const getOpenShiftInfoForProject = (project: string): Promise => graphqlapi.query(` { project:projectByName(name: "${project}"){ + autoIdle + availability + branches + buildImage + developmentBuildPriority + envVariables { + name + scope + value + } + gitUrl id - organization openshift { ...${deployTargetMinimalFragment} } - autoIdle - branches - pullrequests - availability - gitUrl - privateKey - subfolder - routerPattern openshiftProjectPattern + organization + privateKey + productionAlias + productionBuildPriority productionEnvironment productionRoutes - productionAlias + pullrequests + routerPattern + sharedBaasBucket + standbyAlias standbyProductionEnvironment standbyRoutes - standbyAlias - productionBuildPriority storageCalc - developmentBuildPriority - buildImage - sharedBaasBucket - envVariables { - name - value - scope - } + subfolder } } `); @@ -1332,6 +1335,17 @@ fragment on Deployment { } `); +type DeployTargetMinimalFragment = Pick< + Kubernetes, + | 'id' + | 'name' + | 'routerPattern' + | 'buildImage' + | 'disabled' + | 'sharedBaasBucketName' + | 'monitoringConfig' +>; + const deployTargetMinimalFragment = graphqlapi.createFragment(` fragment on Openshift { id diff --git a/node-packages/commons/src/deploy-tasks.ts b/node-packages/commons/src/deploy-tasks.ts index a9ed58a6e7..7a69309487 100644 --- a/node-packages/commons/src/deploy-tasks.ts +++ b/node-packages/commons/src/deploy-tasks.ts @@ -18,13 +18,6 @@ class NoNeedToDeployBranch extends Error { } } -class UnknownActiveSystem extends Error { - constructor(message) { - super(message); - this.name = 'UnknownActiveSystem'; - } -} - /* this function handles deploying a branch */ @@ -42,35 +35,14 @@ const deployBranch = async function(data: any) { switch (branchesRegex) { case undefined: case null: - logger.debug( - `projectName: ${projectName}, branchName: ${branchName}, no branches defined in active system, assuming we want all of them` - ); - switch (project.activeSystemsDeploy) { - case 'lagoon_controllerBuildDeploy': - deployData.deployTarget = deployTarget - const buildDeployData = await getControllerBuildData(deployData); - const sendTasks = await sendToLagoonTasks(buildDeployData.spec.project.deployTarget+':builddeploy', buildDeployData); - return true - default: - throw new UnknownActiveSystem( - `Unknown active system '${project.activeSystemsDeploy}' for task 'deploy' in for project ${projectName}` - ); - } case 'true': logger.debug( `projectName: ${projectName}, branchName: ${branchName}, all branches active, therefore deploying` ); - switch (project.activeSystemsDeploy) { - case 'lagoon_controllerBuildDeploy': - deployData.deployTarget = deployTarget - const buildDeployData = await getControllerBuildData(deployData); - const sendTasks = await sendToLagoonTasks(buildDeployData.spec.project.deployTarget+':builddeploy', buildDeployData); - return true - default: - throw new UnknownActiveSystem( - `Unknown active system '${project.activeSystemsDeploy}' for task 'deploy' in for project ${projectName}` - ); - } + deployData.deployTarget = deployTarget + const buildDeployData = await getControllerBuildData(deployData); + await sendToLagoonTasks(buildDeployData.spec.project.deployTarget+':builddeploy', buildDeployData); + return true case 'false': logger.debug( `projectName: ${projectName}, branchName: ${branchName}, branch deployments disabled` @@ -85,18 +57,11 @@ const deployBranch = async function(data: any) { logger.debug( `projectName: ${projectName}, branchName: ${branchName}, regex ${branchesRegex} matched branchname, starting deploy` ); - switch (project.activeSystemsDeploy) { - case 'lagoon_controllerBuildDeploy': - // controllers uses a different message than the other services, so we need to source it here - deployData.deployTarget = deployTarget - const buildDeployData = await getControllerBuildData(deployData); - const sendTasks = await sendToLagoonTasks(buildDeployData.spec.project.deployTarget+':builddeploy', buildDeployData); - return true - default: - throw new UnknownActiveSystem( - `Unknown active system '${project.activeSystemsDeploy}' for task 'deploy' in for project ${projectName}` - ); - } + // controllers uses a different message than the other services, so we need to source it here + deployData.deployTarget = deployTarget + const buildDeployData = await getControllerBuildData(deployData); + await sendToLagoonTasks(buildDeployData.spec.project.deployTarget+':builddeploy', buildDeployData); + return true } logger.debug( `projectName: ${projectName}, branchName: ${branchName}, regex ${branchesRegex} did not match branchname, not deploying` @@ -124,35 +89,14 @@ const deployPullrequest = async function(data: any) { switch (pullrequestRegex) { case undefined: case null: - logger.debug( - `projectName: ${projectName}, pullrequest: ${branchName}, no pullrequest defined in active system, assuming we want all of them` - ); - switch (project.activeSystemsDeploy) { - case 'lagoon_controllerBuildDeploy': - deployData.deployTarget = deployTarget - const buildDeployData = await getControllerBuildData(deployData); - const sendTasks = await sendToLagoonTasks(buildDeployData.spec.project.deployTarget+':builddeploy', buildDeployData); - return true - default: - throw new UnknownActiveSystem( - `Unknown active system '${project.activeSystemsDeploy}' for task 'deploy' in for project ${projectName}` - ); - } case 'true': logger.debug( `projectName: ${projectName}, pullrequest: ${branchName}, all pullrequest active, therefore deploying` ); - switch (project.activeSystemsDeploy) { - case 'lagoon_controllerBuildDeploy': - deployData.deployTarget = deployTarget - const buildDeployData = await getControllerBuildData(deployData); - const sendTasks = await sendToLagoonTasks(buildDeployData.spec.project.deployTarget+':builddeploy', buildDeployData); - return true - default: - throw new UnknownActiveSystem( - `Unknown active system '${project.activeSystemsDeploy}' for task 'deploy' in for project ${projectName}` - ); - } + deployData.deployTarget = deployTarget + const buildDeployData = await getControllerBuildData(deployData); + await sendToLagoonTasks(buildDeployData.spec.project.deployTarget+':builddeploy', buildDeployData); + return true case 'false': logger.debug( `projectName: ${projectName}, pullrequest: ${branchName}, pullrequest deployments disabled` @@ -167,18 +111,11 @@ const deployPullrequest = async function(data: any) { logger.debug( `projectName: ${projectName}, pullrequest: ${branchName}, regex ${pullrequestRegex} matched PR title '${pullrequestTitle}', starting deploy` ); - switch (project.activeSystemsDeploy) { - case 'lagoon_controllerBuildDeploy': - // controllers uses a different message than the other services, so we need to source it here - deployData.deployTarget = deployTarget - const buildDeployData = await getControllerBuildData(deployData); - const sendTasks = await sendToLagoonTasks(buildDeployData.spec.project.deployTarget+':builddeploy', buildDeployData); - return true - default: - throw new UnknownActiveSystem( - `Unknown active system '${project.activeSystemsDeploy}' for task 'deploy' in for project ${projectName}` - ); - } + // controllers uses a different message than the other services, so we need to source it here + deployData.deployTarget = deployTarget + const buildDeployData = await getControllerBuildData(deployData); + await sendToLagoonTasks(buildDeployData.spec.project.deployTarget+':builddeploy', buildDeployData); + return true } logger.debug( `projectName: ${projectName}, branchName: ${branchName}, regex ${pullrequestRegex} did not match PR title, not deploying` diff --git a/node-packages/commons/src/tasks.ts b/node-packages/commons/src/tasks.ts index 741a839ffb..7e538a3fe2 100644 --- a/node-packages/commons/src/tasks.ts +++ b/node-packages/commons/src/tasks.ts @@ -7,25 +7,26 @@ import { import { ConfirmChannel, ConsumeMessage } from 'amqplib'; import { logger } from './logs/local-logger'; import { - getActiveSystemForProject, + EnvKeyValue, + EnvVariableScope, + Kubernetes, + Project, getEnvironmentsForProject, getOpenShiftInfoForProject, getOpenShiftInfoForEnvironment, - getDeployTargetConfigsForProject, getEnvironmentByIdWithVariables, addOrUpdateEnvironment, getEnvironmentByName, addDeployment, - Project, - DeployTarget, getOrganizationByIdWithEnvs, - getOrganizationById + getOrganizationById, } from './api'; import { deployTargetBranches, deployTargetPullrequest, deployTargetPromote } from './deploy-tasks'; +import { InternalEnvVariableScope } from './types'; import sha1 from 'sha1'; import crypto from 'crypto'; import moment from 'moment'; @@ -126,20 +127,6 @@ const overwriteActiveStandbyTaskImage = process.env.OVERWRITE_ACTIVESTANDBY_TASK const jwtSecretString = process.env.JWTSECRET || "super-secret-string" const projectSeedString = process.env.PROJECTSEED || "super-secret-string" -class UnknownActiveSystem extends Error { - constructor(message) { - super(message); - this.name = 'UnknownActiveSystem'; - } -} - -class CannotDeployWithDeployTargetConfigs extends Error { - constructor(message) { - super(message); - this.name = 'CannotDeployWithDeployTargetConfigs'; - } -} - class NoNeedToRemoveBranch extends Error { constructor(message) { super(message); @@ -681,8 +668,24 @@ enum bulkType { } export const getEnvironmentsRouterPatternAndVariables = async function name( - project: Project, environment: any, deployTarget: DeployTarget, - bulkId: string, bulkName: string, buildPriority: number, buildVariables: any, bulkTask: bulkType) { + project: Pick< + Project, + 'routerPattern' | 'sharedBaasBucket' | 'organization' + > & { + openshift: Pick; + envVariables: Pick[]; + }, + environment: any, + deployTarget: Pick, + bulkId: string, + bulkName: string, + buildPriority: number, + buildVariables: any, + bulkTask: bulkType +): Promise<[string, string, string]> { + let projectVars: Array & { + scope: EnvVariableScope | InternalEnvVariableScope; + }> = [...project.envVariables]; // set routerpattern to the routerpattern of what is defined in the project scope openshift var routerPattern = project.openshift.routerPattern @@ -696,29 +699,56 @@ export const getEnvironmentsRouterPatternAndVariables = async function name( // if a project has a routerpattern defined, use it. `null` is not valid here routerPattern = project.routerPattern } - // append the routerpattern to the projects variables - // use a scope of `internal_system` which isn't available to the actual API to be added via mutations - // this way variables or new functionality can be passed into lagoon builds using the existing variables mechanism - // avoiding the needs to hardcode them into the spec to then be consumed by the build-deploy controller - project.envVariables.push({"name":"LAGOON_SYSTEM_ROUTER_PATTERN", "value":routerPattern, "scope":"internal_system"}) - // append the `LAGOON_SYSTEM_CORE_VERSION` variable as an `internal_system` variable that can be consumed by builds and - // is not user configurable, this value will eventually be consumed by `build-deploy-tool` to be able to reject - // builds that are not of a supported version of lagoon - project.envVariables.push({"name":"LAGOON_SYSTEM_CORE_VERSION", "value":lagoonVersion, "scope":"internal_system"}) + + projectVars = [ + ...projectVars, + // append the routerpattern to the projects variables + // use a scope of `internal_system` which isn't available to the actual API to be added via mutations + // this way variables or new functionality can be passed into lagoon builds using the existing variables mechanism + // avoiding the needs to hardcode them into the spec to then be consumed by the build-deploy controller + { + name: 'LAGOON_SYSTEM_ROUTER_PATTERN', + value: routerPattern, + scope: InternalEnvVariableScope.INTERNAL_SYSTEM + }, + // append the `LAGOON_SYSTEM_CORE_VERSION` variable as an `internal_system` variable that can be consumed by builds and + // is not user configurable, this value will eventually be consumed by `build-deploy-tool` to be able to reject + // builds that are not of a supported version of lagoon + { + name: "LAGOON_SYSTEM_CORE_VERSION", + value: lagoonVersion, + scope: InternalEnvVariableScope.INTERNAL_SYSTEM + } + ]; + // if the project is configured with a shared baas bucket if (project.sharedBaasBucket) { // we only want the shared baas bucket here if one is defined let [sharedBaasBucket, shared] = await getBaasBucketName(project, deployTarget) if (shared) { - project.envVariables.push({"name":"LAGOON_SYSTEM_PROJECT_SHARED_BUCKET", "value":sharedBaasBucket, "scope":"internal_system"}) + projectVars = [ + ...projectVars, + { + name: "LAGOON_SYSTEM_PROJECT_SHARED_BUCKET", + value: sharedBaasBucket, + scope: InternalEnvVariableScope.INTERNAL_SYSTEM + } + ]; } } if (project.organization) { // check the environment quota, this prevents environments being deployed by the api or webhooks const curOrg = await getOrganizationById(project.organization); - project.envVariables.push({"name":"LAGOON_ROUTE_QUOTA", "value":`"${curOrg.quotaRoute}"`, "scope":"internal_system"}) + projectVars = [ + ...projectVars, + { + name: "LAGOON_ROUTE_QUOTA", + value: `"${curOrg.quotaRoute}"`, + scope: InternalEnvVariableScope.INTERNAL_SYSTEM + } + ]; } // handle any bulk deploy related injections here @@ -727,29 +757,61 @@ export const getEnvironmentsRouterPatternAndVariables = async function name( case bulkType.Task: varPrefix = "LAGOON_BULK_TASK" if (buildPriority != null) { - project.envVariables.push({"name":"LAGOON_TASK_PRIORITY", "value":buildPriority.toString(), "scope":"build"}) + projectVars = [ + ...projectVars, + { + name: "LAGOON_TASK_PRIORITY", + value: buildPriority.toString(), + scope: EnvVariableScope.BUILD + } + ]; } break; default: if (buildPriority != null) { - project.envVariables.push({"name":"LAGOON_BUILD_PRIORITY", "value":buildPriority.toString(), "scope":"build"}) + projectVars = [ + ...projectVars, + { + name: "LAGOON_BUILD_PRIORITY", + value: buildPriority.toString(), + scope: EnvVariableScope.BUILD + } + ]; } break; } if (bulkId != "" && bulkId != null) { // if this is a bulk deploy, add the associated bulk deploy build scope variables - project.envVariables.push({"name": varPrefix, "value":"true", "scope":"build"}) - project.envVariables.push({"name": varPrefix+"_ID", "value":bulkId, "scope":"build"}) + projectVars = [ + ...projectVars, + { + name: varPrefix, + value: "true", + scope: EnvVariableScope.BUILD + }, + { + name: `${varPrefix}_ID`, + value: bulkId, + scope: EnvVariableScope.BUILD + } + ]; } if (bulkName != "" && bulkName != null) { - project.envVariables.push({"name": varPrefix+"_NAME", "value":bulkName, "scope":"build"}) + projectVars = [ + ...projectVars, + { + name: `${varPrefix}_NAME`, + value: bulkName, + scope: EnvVariableScope.BUILD + } + ]; } // end bulk related injections let lagoonEnvironmentVariables = environment.envVariables || [] if (buildVariables != null ) { // add the build `scope` to all the incoming build variables for a specific build - const scopedBuildVariables = buildVariables.map(v => ({...v, scope: 'build'})) + const scopedBuildVariables = buildVariables.map(v => ({...v, scope: EnvVariableScope.BUILD})) // check for buildvariables being passed in // these need to be merged on top of environment level variables // handle that here @@ -758,10 +820,10 @@ export const getEnvironmentsRouterPatternAndVariables = async function name( // encode some values so they get sent to the controllers nicely - const envVars = new Buffer(JSON.stringify(lagoonEnvironmentVariables)).toString('base64') - const projectVars = new Buffer(JSON.stringify(project.envVariables)).toString('base64') + const envVarsEncoded = new Buffer(JSON.stringify(lagoonEnvironmentVariables)).toString('base64') + const projectVarsEncoded = new Buffer(JSON.stringify(projectVars)).toString('base64') - return [routerPattern, envVars, projectVars] + return [routerPattern, envVarsEncoded, projectVarsEncoded] } /* @@ -778,7 +840,8 @@ export const createDeployTask = async function(deployData: any) { pullrequestTitle } = deployData; - const project = await getActiveSystemForProject(projectName, 'Deploy'); + const result = await getOpenShiftInfoForProject(projectName); + const project = result.project; const environments = await getEnvironmentsForProject(projectName); if (project.organization) { @@ -800,104 +863,90 @@ export const createDeployTask = async function(deployData: any) { // production_environment: 'master', // environments: [ { name: 'develop', environment_type: 'development' }, [Object] ] } } - if (typeof project.activeSystemsDeploy === 'undefined') { - throw new UnknownActiveSystem( - `No active system for tasks 'deploy' in for project ${projectName}` + // we want to limit production environments, without making it configurable currently + var productionEnvironmentsLimit = 2; + + // we want to make sure we can deploy the `production` env, and also the env defined as standby + if ( + environments.project.productionEnvironment === branchName || + environments.project.standbyProductionEnvironment === branchName + ) { + // get a list of production environments + const prod_environments = environments.project.environments + .filter(e => e.environmentType === 'production') + .map(e => e.name); + logger.debug( + `projectName: ${projectName}, branchName: ${branchName}, existing environments are ${prod_environments}` ); - } - switch (project.activeSystemsDeploy) { - case 'lagoon_controllerBuildDeploy': - // we want to limit production environments, without making it configurable currently - var productionEnvironmentsLimit = 2; - - // we want to make sure we can deploy the `production` env, and also the env defined as standby - if ( - environments.project.productionEnvironment === branchName || - environments.project.standbyProductionEnvironment === branchName - ) { - // get a list of production environments - const prod_environments = environments.project.environments - .filter(e => e.environmentType === 'production') - .map(e => e.name); + if (prod_environments.length >= productionEnvironmentsLimit) { + if (prod_environments.find(i => i === branchName)) { logger.debug( - `projectName: ${projectName}, branchName: ${branchName}, existing environments are ${prod_environments}` + `projectName: ${projectName}, branchName: ${branchName}, environment already exists, no environment limits considered` ); - - if (prod_environments.length >= productionEnvironmentsLimit) { - if (prod_environments.find(i => i === branchName)) { - logger.debug( - `projectName: ${projectName}, branchName: ${branchName}, environment already exists, no environment limits considered` - ); - } else { - throw new EnvironmentLimit( - `'${branchName}' would exceed the configured limit of ${productionEnvironmentsLimit} production environments for project ${projectName}` - ); - } - } } else { - // get a list of non-production environments - const dev_environments = environments.project.environments - .filter(e => e.environmentType === 'development') - .map(e => e.name); - logger.debug( - `projectName: ${projectName}, branchName: ${branchName}, existing environments are ${dev_environments}` + throw new EnvironmentLimit( + `'${branchName}' would exceed the configured limit of ${productionEnvironmentsLimit} production environments for project ${projectName}` ); - - if ( - environments.project.developmentEnvironmentsLimit !== null && - dev_environments.length >= - environments.project.developmentEnvironmentsLimit - ) { - if (dev_environments.find(i => i === branchName)) { - logger.debug( - `projectName: ${projectName}, branchName: ${branchName}, environment already exists, no environment limits considered` - ); - } else { - throw new EnvironmentLimit( - `'${branchName}' would exceed the configured limit of ${environments.project.developmentEnvironmentsLimit} development environments for project ${projectName}` - ); - } - } } + } + } else { + // get a list of non-production environments + const dev_environments = environments.project.environments + .filter(e => e.environmentType === 'development') + .map(e => e.name); + logger.debug( + `projectName: ${projectName}, branchName: ${branchName}, existing environments are ${dev_environments}` + ); - if (type === 'branch') { - // use deployTargetBranches function to handle - let lagoonData = { - projectId: environments.project.id, - projectName, - branchName, - project, - deployData - } - try { - let result = deployTargetBranches(lagoonData) - return result - } catch (error) { - throw error - } - } else if (type === 'pullrequest') { - // use deployTargetPullrequest function to handle - let lagoonData = { - projectId: environments.project.id, - projectName, - branchName, - project, - pullrequestTitle, - deployData - } - try { - let result = deployTargetPullrequest(lagoonData) - return result - } catch (error) { - throw error - } + if ( + environments.project.developmentEnvironmentsLimit !== null && + dev_environments.length >= + environments.project.developmentEnvironmentsLimit + ) { + if (dev_environments.find(i => i === branchName)) { + logger.debug( + `projectName: ${projectName}, branchName: ${branchName}, environment already exists, no environment limits considered` + ); + } else { + throw new EnvironmentLimit( + `'${branchName}' would exceed the configured limit of ${environments.project.developmentEnvironmentsLimit} development environments for project ${projectName}` + ); } - break; - default: - throw new UnknownActiveSystem( - `Unknown active system '${project.activeSystemsDeploy}' for task 'deploy' in for project ${projectName}` - ); + } + } + + if (type === 'branch') { + // use deployTargetBranches function to handle + let lagoonData = { + projectId: environments.project.id, + projectName, + branchName, + project, + deployData + } + try { + let result = deployTargetBranches(lagoonData) + return result + } catch (error) { + throw error + } + } else if (type === 'pullrequest') { + // use deployTargetPullrequest function to handle + let lagoonData = { + projectId: environments.project.id, + projectName, + branchName, + project, + pullrequestTitle, + deployData + } + try { + let result = deployTargetPullrequest(lagoonData) + return result + } catch (error) { + throw error + } } } @@ -909,27 +958,15 @@ export const createPromoteTask = async function(promoteData: any) { // type, } = promoteData; - const project = await getActiveSystemForProject(projectName, 'Promote'); - - if (typeof project.activeSystemsPromote === 'undefined') { - throw new UnknownActiveSystem( - `No active system for tasks 'deploy' in for project ${projectName}` - ); - } + const result = await getOpenShiftInfoForProject(projectName); + const project = result.project; - switch (project.activeSystemsPromote) { - case 'lagoon_controllerBuildDeploy': - // use deployTargetPromote function to handle - let lagoonData = { - projectId: project.id, - promoteData - } - return deployTargetPromote(lagoonData) - default: - throw new UnknownActiveSystem( - `Unknown active system '${project.activeSystemsPromote}' for task 'deploy' in for project ${projectName}` - ); + // use deployTargetPromote function to handle + let lagoonData = { + projectId: project.id, + promoteData } + return deployTargetPromote(lagoonData) } export const createRemoveTask = async function(removeData: any) { @@ -960,114 +997,95 @@ export const createRemoveTask = async function(removeData: any) { } } - const project = await getActiveSystemForProject(projectName, 'Remove'); + if (type === 'branch') { + let environmentId = 0; + // Check to ensure the environment actually exists. + let foundEnvironment = false; + allEnvironments.project.environments.forEach(function( + environment, + index + ) { + if (environment.name === branch) { + foundEnvironment = true; + environmentId = environment.id; + } + }); - if (typeof project.activeSystemsRemove === 'undefined') { - throw new UnknownActiveSystem( - `No active system for tasks 'remove' in for project ${projectName}` + if (!foundEnvironment) { + logger.debug( + `projectName: ${projectName}, branchName: ${branch}, no environment found.` + ); + throw new NoNeedToRemoveBranch( + 'Branch environment does not exist, no need to remove anything.' + ); + } + // consume the deploytarget from the environment now + const result = await getOpenShiftInfoForEnvironment(environmentId); + const deployTarget = result.environment.openshift.name + logger.debug( + `projectName: ${projectName}, branchName: ${branchName}. Removing branch environment.` ); - } + // use the targetname as the routing key with the action + return sendToLagoonTasks(deployTarget+":remove", removeData); + } else if (type === 'pullrequest') { + // Work out the branch name from the PR number. + let branchName = 'pr-' + pullrequestNumber; + removeData.branchName = 'pr-' + pullrequestNumber; + + let environmentId = 0; + // Check to ensure the environment actually exists. + let foundEnvironment = false; + allEnvironments.project.environments.forEach(function( + environment, + index + ) { + if (environment.name === branchName) { + foundEnvironment = true; + environmentId = environment.id; + } + }); - switch (project.activeSystemsRemove) { - // removed `openshift` and `kubernetes` remove functionality, these services no longer exist in Lagoon - // handle removals using the controllers, send the message to our specific target cluster queue - case 'lagoon_controllerRemove': - if (type === 'branch') { - let environmentId = 0; - // Check to ensure the environment actually exists. - let foundEnvironment = false; - allEnvironments.project.environments.forEach(function( - environment, - index - ) { - if (environment.name === branch) { - foundEnvironment = true; - environmentId = environment.id; - } - }); - - if (!foundEnvironment) { - logger.debug( - `projectName: ${projectName}, branchName: ${branch}, no environment found.` - ); - throw new NoNeedToRemoveBranch( - 'Branch environment does not exist, no need to remove anything.' - ); - } - // consume the deploytarget from the environment now - const result = await getOpenShiftInfoForEnvironment(environmentId); - const deployTarget = result.environment.openshift.name - logger.debug( - `projectName: ${projectName}, branchName: ${branchName}. Removing branch environment.` - ); - // use the targetname as the routing key with the action - return sendToLagoonTasks(deployTarget+":remove", removeData); - } else if (type === 'pullrequest') { - // Work out the branch name from the PR number. - let branchName = 'pr-' + pullrequestNumber; - removeData.branchName = 'pr-' + pullrequestNumber; - - let environmentId = 0; - // Check to ensure the environment actually exists. - let foundEnvironment = false; - allEnvironments.project.environments.forEach(function( - environment, - index - ) { - if (environment.name === branchName) { - foundEnvironment = true; - environmentId = environment.id; - } - }); - - if (!foundEnvironment) { - logger.debug( - `projectName: ${projectName}, pullrequest: ${branchName}, no pullrequest found.` - ); - throw new NoNeedToRemoveBranch( - 'Pull Request environment does not exist, no need to remove anything.' - ); - } - // consume the deploytarget from the environment now - const result = await getOpenShiftInfoForEnvironment(environmentId); - const deployTarget = result.environment.openshift.name - logger.debug( - `projectName: ${projectName}, pullrequest: ${branchName}. Removing pullrequest environment.` - ); - return sendToLagoonTasks(deployTarget+":remove", removeData); - } else if (type === 'promote') { - let environmentId = 0; - // Check to ensure the environment actually exists. - let foundEnvironment = false; - allEnvironments.project.environments.forEach(function( - environment, - index - ) { - if (environment.name === branch) { - foundEnvironment = true; - environmentId = environment.id; - } - }); - - if (!foundEnvironment) { - logger.debug( - `projectName: ${projectName}, branchName: ${branch}, no environment found.` - ); - throw new NoNeedToRemoveBranch( - 'Branch environment does not exist, no need to remove anything.' - ); - } - // consume the deploytarget from the environment now - const result = await getOpenShiftInfoForEnvironment(environmentId); - const deployTarget = result.environment.openshift.name - return sendToLagoonTasks(deployTarget+":remove", removeData); + if (!foundEnvironment) { + logger.debug( + `projectName: ${projectName}, pullrequest: ${branchName}, no pullrequest found.` + ); + throw new NoNeedToRemoveBranch( + 'Pull Request environment does not exist, no need to remove anything.' + ); + } + // consume the deploytarget from the environment now + const result = await getOpenShiftInfoForEnvironment(environmentId); + const deployTarget = result.environment.openshift.name + logger.debug( + `projectName: ${projectName}, pullrequest: ${branchName}. Removing pullrequest environment.` + ); + return sendToLagoonTasks(deployTarget+":remove", removeData); + } else if (type === 'promote') { + let environmentId = 0; + // Check to ensure the environment actually exists. + let foundEnvironment = false; + allEnvironments.project.environments.forEach(function( + environment, + index + ) { + if (environment.name === branch) { + foundEnvironment = true; + environmentId = environment.id; } - break; + }); - default: - throw new UnknownActiveSystem( - `Unknown active system '${project.activeSystemsRemove}' for task 'remove' in for project ${projectName}` + if (!foundEnvironment) { + logger.debug( + `projectName: ${projectName}, branchName: ${branch}, no environment found.` + ); + throw new NoNeedToRemoveBranch( + 'Branch environment does not exist, no need to remove anything.' ); + } + // consume the deploytarget from the environment now + const result = await getOpenShiftInfoForEnvironment(environmentId); + const deployTarget = result.environment.openshift.name + return sendToLagoonTasks(deployTarget+":remove", removeData); } } @@ -1111,7 +1129,12 @@ export const getTaskProjectEnvironmentVariables =async (projectName: string, env return [projectVars, envVars] } -export const getBaasBucketName = async (project: Project, deploytarget: DeployTarget) => { +export const getBaasBucketName = async ( + project: Pick & { + envVariables: Pick[]; + }, + deploytarget: Pick +) => { // logic to check if the project is defined with a shared bucket or has a bucket name override let sharedBaasBucketName let baasBucketName @@ -1141,8 +1164,6 @@ export const getBaasBucketName = async (project: Project, deploytarget: DeployTa export const createTaskTask = async function(taskData: any) { const { project } = taskData; - const projectSystem = await getActiveSystemForProject(project.name, 'Task'); - // inject variables into tasks the same way it is in builds const [_, envVars, projectVars] = await getTaskProjectEnvironmentVariables( project.name, @@ -1162,25 +1183,11 @@ export const createTaskTask = async function(taskData: any) { taskData.project.organization = organization } - if (typeof projectSystem.activeSystemsTask === 'undefined') { - throw new UnknownActiveSystem( - `No active system for 'task' for project ${project.name}` - ); - } - - switch (projectSystem.activeSystemsTask) { - case 'lagoon_controllerJob': - // since controllers queues are named, we have to send it to the right tasks queue - // do that here by querying which deploytarget the environment uses - const result = await getOpenShiftInfoForEnvironment(taskData.environment.id); - const deployTarget = result.environment.openshift.name - return sendToLagoonTasks(deployTarget+":jobs", taskData); - - default: - throw new UnknownActiveSystem( - `Unknown active system '${projectSystem.activeSystemsTask}' for 'task' for project ${project.name}` - ); - } + // since controllers queues are named, we have to send it to the right tasks queue + // do that here by querying which deploytarget the environment uses + const result = await getOpenShiftInfoForEnvironment(taskData.environment.id); + const deployTarget = result.environment.openshift.name + return sendToLagoonTasks(deployTarget+":jobs", taskData); } export const createMiscTask = async function(taskData: any) { @@ -1189,208 +1196,197 @@ export const createMiscTask = async function(taskData: any) { data: { project } } = taskData; - const data = await getActiveSystemForProject(project.name, 'Misc'); - - let updatedKey = key; - let taskId = ''; - switch (data.activeSystemsMisc) { - case 'lagoon_controllerMisc': - // handle any controller based misc tasks - updatedKey = `deploytarget:${key}`; - taskId = 'misc-kubernetes'; - // determine the deploy target (openshift/kubernetes) for the task to go to - // we get this from the environment - const result = await getOpenShiftInfoForEnvironment(taskData.data.environment.id); - const deployTarget = result.environment.openshift.name - // this is the json structure for sending a misc task to the controller - // there are some additional bits that can be adjusted, and these are done in the switch below on `updatedKey` - var miscTaskData: any = { - misc: {}, - key: updatedKey, - environment: { - name: taskData.data.environment.name, - openshiftProjectName: taskData.data.environment.openshiftProjectName - }, - project: { - name: taskData.data.project.name - }, - task: taskData.data.task, - advancedTask: {} + // handle any controller based misc tasks + let updatedKey = `deploytarget:${key}`; + let taskId = 'misc-kubernetes'; + // determine the deploy target (openshift/kubernetes) for the task to go to + // we get this from the environment + const result = await getOpenShiftInfoForEnvironment(taskData.data.environment.id); + const deployTarget = result.environment.openshift.name + // this is the json structure for sending a misc task to the controller + // there are some additional bits that can be adjusted, and these are done in the switch below on `updatedKey` + var miscTaskData: any = { + misc: {}, + key: updatedKey, + environment: { + name: taskData.data.environment.name, + openshiftProjectName: taskData.data.environment.openshiftProjectName + }, + project: { + name: taskData.data.project.name + }, + task: taskData.data.task, + advancedTask: {} + } + switch (updatedKey) { + case 'deploytarget:restic:backup:restore': + // Handle setting up the configuration for a restic restoration task + const randRestoreId = Math.random().toString(36).substring(7); + const restoreName = `restore-${R.slice(0, 7, taskData.data.backup.backupId)}-${randRestoreId}`; + // Parse out the baasBucketName for any migrated projects + // check if the project is configured for a shared baas bucket + let [baasBucketName, shared] = await getBaasBucketName(result.environment.project, result.environment.openshift) + if (shared) { + // if it is a shared bucket, add the repo key to it too for restores + baasBucketName = `${baasBucketName}/baas-${makeSafe(taskData.data.project.name)}` + } + // Handle custom backup configurations + let lagoonBaasCustomBackupEndpoint = result.environment.project.envVariables.find(obj => { + return obj.name === "LAGOON_BAAS_CUSTOM_BACKUP_ENDPOINT" + }) + if (lagoonBaasCustomBackupEndpoint) { + lagoonBaasCustomBackupEndpoint = lagoonBaasCustomBackupEndpoint.value + } + let lagoonBaasCustomBackupBucket = result.environment.project.envVariables.find(obj => { + return obj.name === "LAGOON_BAAS_CUSTOM_BACKUP_BUCKET" + }) + if (lagoonBaasCustomBackupBucket) { + lagoonBaasCustomBackupBucket = lagoonBaasCustomBackupBucket.value + } + let lagoonBaasCustomBackupAccessKey = result.environment.project.envVariables.find(obj => { + return obj.name === "LAGOON_BAAS_CUSTOM_BACKUP_ACCESS_KEY" + }) + if (lagoonBaasCustomBackupAccessKey) { + lagoonBaasCustomBackupAccessKey = lagoonBaasCustomBackupAccessKey.value + } + let lagoonBaasCustomBackupSecretKey = result.environment.project.envVariables.find(obj => { + return obj.name === "LAGOON_BAAS_CUSTOM_BACKUP_SECRET_KEY" + }) + if (lagoonBaasCustomBackupSecretKey) { + lagoonBaasCustomBackupSecretKey = lagoonBaasCustomBackupSecretKey.value } - switch (updatedKey) { - case 'deploytarget:restic:backup:restore': - // Handle setting up the configuration for a restic restoration task - const randRestoreId = Math.random().toString(36).substring(7); - const restoreName = `restore-${R.slice(0, 7, taskData.data.backup.backupId)}-${randRestoreId}`; - // Parse out the baasBucketName for any migrated projects - // check if the project is configured for a shared baas bucket - let [baasBucketName, shared] = await getBaasBucketName(result.environment.project, result.environment.openshift) - if (shared) { - // if it is a shared bucket, add the repo key to it too for restores - baasBucketName = `${baasBucketName}/baas-${makeSafe(taskData.data.project.name)}` - } - // Handle custom backup configurations - let lagoonBaasCustomBackupEndpoint = result.environment.project.envVariables.find(obj => { - return obj.name === "LAGOON_BAAS_CUSTOM_BACKUP_ENDPOINT" - }) - if (lagoonBaasCustomBackupEndpoint) { - lagoonBaasCustomBackupEndpoint = lagoonBaasCustomBackupEndpoint.value - } - let lagoonBaasCustomBackupBucket = result.environment.project.envVariables.find(obj => { - return obj.name === "LAGOON_BAAS_CUSTOM_BACKUP_BUCKET" - }) - if (lagoonBaasCustomBackupBucket) { - lagoonBaasCustomBackupBucket = lagoonBaasCustomBackupBucket.value - } - let lagoonBaasCustomBackupAccessKey = result.environment.project.envVariables.find(obj => { - return obj.name === "LAGOON_BAAS_CUSTOM_BACKUP_ACCESS_KEY" - }) - if (lagoonBaasCustomBackupAccessKey) { - lagoonBaasCustomBackupAccessKey = lagoonBaasCustomBackupAccessKey.value - } - let lagoonBaasCustomBackupSecretKey = result.environment.project.envVariables.find(obj => { - return obj.name === "LAGOON_BAAS_CUSTOM_BACKUP_SECRET_KEY" - }) - if (lagoonBaasCustomBackupSecretKey) { - lagoonBaasCustomBackupSecretKey = lagoonBaasCustomBackupSecretKey.value - } - let backupS3Config = {} - if (lagoonBaasCustomBackupEndpoint && lagoonBaasCustomBackupBucket && lagoonBaasCustomBackupAccessKey && lagoonBaasCustomBackupSecretKey) { - backupS3Config = { - endpoint: lagoonBaasCustomBackupEndpoint, - bucket: lagoonBaasCustomBackupBucket, - accessKeyIDSecretRef: { - name: "lagoon-baas-custom-backup-credentials", - key: "access-key" - }, - secretAccessKeySecretRef: { - name: "lagoon-baas-custom-backup-credentials", - key: "secret-key" - } - } - } else { - backupS3Config = { - bucket: baasBucketName ? baasBucketName : `baas-${makeSafe(taskData.data.project.name)}` - } + let backupS3Config = {} + if (lagoonBaasCustomBackupEndpoint && lagoonBaasCustomBackupBucket && lagoonBaasCustomBackupAccessKey && lagoonBaasCustomBackupSecretKey) { + backupS3Config = { + endpoint: lagoonBaasCustomBackupEndpoint, + bucket: lagoonBaasCustomBackupBucket, + accessKeyIDSecretRef: { + name: "lagoon-baas-custom-backup-credentials", + key: "access-key" + }, + secretAccessKeySecretRef: { + name: "lagoon-baas-custom-backup-credentials", + key: "secret-key" } + } + } else { + backupS3Config = { + bucket: baasBucketName ? baasBucketName : `baas-${makeSafe(taskData.data.project.name)}` + } + } - // Handle custom restore configurations - let lagoonBaasCustomRestoreEndpoint = result.environment.project.envVariables.find(obj => { - return obj.name === "LAGOON_BAAS_CUSTOM_RESTORE_ENDPOINT" - }) - if (lagoonBaasCustomRestoreEndpoint) { - lagoonBaasCustomRestoreEndpoint = lagoonBaasCustomRestoreEndpoint.value - } - let lagoonBaasCustomRestoreBucket = result.environment.project.envVariables.find(obj => { - return obj.name === "LAGOON_BAAS_CUSTOM_RESTORE_BUCKET" - }) - if (lagoonBaasCustomRestoreBucket) { - lagoonBaasCustomRestoreBucket = lagoonBaasCustomRestoreBucket.value - } - let lagoonBaasCustomRestoreAccessKey = result.environment.project.envVariables.find(obj => { - return obj.name === "LAGOON_BAAS_CUSTOM_RESTORE_ACCESS_KEY" - }) - if (lagoonBaasCustomRestoreAccessKey) { - lagoonBaasCustomRestoreAccessKey = lagoonBaasCustomRestoreAccessKey.value - } - let lagoonBaasCustomRestoreSecretKey = result.environment.project.envVariables.find(obj => { - return obj.name === "LAGOON_BAAS_CUSTOM_RESTORE_SECRET_KEY" - }) - if (lagoonBaasCustomRestoreSecretKey) { - lagoonBaasCustomRestoreSecretKey = lagoonBaasCustomRestoreSecretKey.value - } + // Handle custom restore configurations + let lagoonBaasCustomRestoreEndpoint = result.environment.project.envVariables.find(obj => { + return obj.name === "LAGOON_BAAS_CUSTOM_RESTORE_ENDPOINT" + }) + if (lagoonBaasCustomRestoreEndpoint) { + lagoonBaasCustomRestoreEndpoint = lagoonBaasCustomRestoreEndpoint.value + } + let lagoonBaasCustomRestoreBucket = result.environment.project.envVariables.find(obj => { + return obj.name === "LAGOON_BAAS_CUSTOM_RESTORE_BUCKET" + }) + if (lagoonBaasCustomRestoreBucket) { + lagoonBaasCustomRestoreBucket = lagoonBaasCustomRestoreBucket.value + } + let lagoonBaasCustomRestoreAccessKey = result.environment.project.envVariables.find(obj => { + return obj.name === "LAGOON_BAAS_CUSTOM_RESTORE_ACCESS_KEY" + }) + if (lagoonBaasCustomRestoreAccessKey) { + lagoonBaasCustomRestoreAccessKey = lagoonBaasCustomRestoreAccessKey.value + } + let lagoonBaasCustomRestoreSecretKey = result.environment.project.envVariables.find(obj => { + return obj.name === "LAGOON_BAAS_CUSTOM_RESTORE_SECRET_KEY" + }) + if (lagoonBaasCustomRestoreSecretKey) { + lagoonBaasCustomRestoreSecretKey = lagoonBaasCustomRestoreSecretKey.value + } - let restoreS3Config = {} - if (lagoonBaasCustomRestoreEndpoint && lagoonBaasCustomRestoreBucket && lagoonBaasCustomRestoreAccessKey && lagoonBaasCustomRestoreSecretKey) { - restoreS3Config = { - endpoint: lagoonBaasCustomRestoreEndpoint, - bucket: lagoonBaasCustomRestoreBucket, - accessKeyIDSecretRef: { - name: "lagoon-baas-custom-restore-credentials", - key: "access-key" - }, - secretAccessKeySecretRef: { - name: "lagoon-baas-custom-restore-credentials", - key: "secret-key" - } - } + let restoreS3Config = {} + if (lagoonBaasCustomRestoreEndpoint && lagoonBaasCustomRestoreBucket && lagoonBaasCustomRestoreAccessKey && lagoonBaasCustomRestoreSecretKey) { + restoreS3Config = { + endpoint: lagoonBaasCustomRestoreEndpoint, + bucket: lagoonBaasCustomRestoreBucket, + accessKeyIDSecretRef: { + name: "lagoon-baas-custom-restore-credentials", + key: "access-key" + }, + secretAccessKeySecretRef: { + name: "lagoon-baas-custom-restore-credentials", + key: "secret-key" } + } + } - // generate the restore CRD - const restoreConf = restoreConfig(restoreName, taskData.data.backup.backupId, backupS3Config, restoreS3Config) - //logger.info(restoreConf) - // base64 encode it - const restoreBytes = new Buffer(JSON.stringify(restoreConf).replace(/\\n/g, "\n")).toString('base64') - miscTaskData.misc.miscResource = restoreBytes - break; - case 'deploytarget:task:activestandby': - // handle setting up the task configuration for running the active/standby switch - // this uses the `advanced task` system in the controllers - // generate out custom json payload to send to the advanced task - var jsonPayload: any = { - productionEnvironment: taskData.data.productionEnvironment.name, - standbyEnvironment: taskData.data.environment.name, - sourceNamespace: makeSafe(taskData.data.environment.openshiftProjectName), - destinationNamespace: makeSafe(taskData.data.productionEnvironment.openshiftProjectName) - } - // encode it - const jsonPayloadBytes = new Buffer(JSON.stringify(jsonPayload).replace(/\\n/g, "\n")).toString('base64') - // set the task data up - miscTaskData.advancedTask.JSONPayload = jsonPayloadBytes - // use this image to run the task - let taskImage = "" - // choose which task image to use - if (CI == "true") { - taskImage = "172.17.0.1:5000/lagoon/task-activestandby:latest" - } else if (overwriteActiveStandbyTaskImage) { - // allow to overwrite the image we use via OVERWRITE_ACTIVESTANDBY_TASK_IMAGE env variable - taskImage = overwriteActiveStandbyTaskImage - } else { - taskImage = `uselagoon/task-activestandby:${lagoonVersion}` - } - miscTaskData.advancedTask.runnerImage = taskImage - // miscTaskData.advancedTask.runnerImage = "shreddedbacon/runner:latest" - break; - case 'deploytarget:task:advanced': - // inject variables into advanced tasks the same way it is in builds and standard tasks - const [_, envVars, projectVars] = await getTaskProjectEnvironmentVariables( - taskData.data.project.name, - taskData.data.environment.id - ) - miscTaskData.project.variables = { - project: projectVars, - environment: envVars, - } - miscTaskData.advancedTask = taskData.data.advancedTask - break; - case 'deploytarget:task:cancel': - // task cancellation is just a standard unmodified message - miscTaskData.misc = taskData.data.task - break; - case 'deploytarget:build:cancel': - // build cancellation is just a standard unmodified message - miscTaskData.misc = taskData.data.build - break; - default: - miscTaskData.misc = taskData.data.build - break; + // generate the restore CRD + const restoreConf = restoreConfig(restoreName, taskData.data.backup.backupId, backupS3Config, restoreS3Config) + //logger.info(restoreConf) + // base64 encode it + const restoreBytes = new Buffer(JSON.stringify(restoreConf).replace(/\\n/g, "\n")).toString('base64') + miscTaskData.misc.miscResource = restoreBytes + break; + case 'deploytarget:task:activestandby': + // handle setting up the task configuration for running the active/standby switch + // this uses the `advanced task` system in the controllers + // generate out custom json payload to send to the advanced task + var jsonPayload: any = { + productionEnvironment: taskData.data.productionEnvironment.name, + standbyEnvironment: taskData.data.environment.name, + sourceNamespace: makeSafe(taskData.data.environment.openshiftProjectName), + destinationNamespace: makeSafe(taskData.data.productionEnvironment.openshiftProjectName) } - // send the task to the queue - if (project.organization != null) { - const curOrg = await getOrganizationById(project.organization); - const organization = { - name: curOrg.name, - id: curOrg.id, - } - miscTaskData.project.organization = organization + // encode it + const jsonPayloadBytes = new Buffer(JSON.stringify(jsonPayload).replace(/\\n/g, "\n")).toString('base64') + // set the task data up + miscTaskData.advancedTask.JSONPayload = jsonPayloadBytes + // use this image to run the task + let taskImage = "" + // choose which task image to use + if (CI == "true") { + taskImage = "172.17.0.1:5000/lagoon/task-activestandby:latest" + } else if (overwriteActiveStandbyTaskImage) { + // allow to overwrite the image we use via OVERWRITE_ACTIVESTANDBY_TASK_IMAGE env variable + taskImage = overwriteActiveStandbyTaskImage + } else { + taskImage = `uselagoon/task-activestandby:${lagoonVersion}` } - return sendToLagoonTasks(deployTarget+':misc', miscTaskData); + miscTaskData.advancedTask.runnerImage = taskImage + // miscTaskData.advancedTask.runnerImage = "shreddedbacon/runner:latest" + break; + case 'deploytarget:task:advanced': + // inject variables into advanced tasks the same way it is in builds and standard tasks + const [_, envVars, projectVars] = await getTaskProjectEnvironmentVariables( + taskData.data.project.name, + taskData.data.environment.id + ) + miscTaskData.project.variables = { + project: projectVars, + environment: envVars, + } + miscTaskData.advancedTask = taskData.data.advancedTask + break; + case 'deploytarget:task:cancel': + // task cancellation is just a standard unmodified message + miscTaskData.misc = taskData.data.task + break; + case 'deploytarget:build:cancel': + // build cancellation is just a standard unmodified message + miscTaskData.misc = taskData.data.build + break; default: + miscTaskData.misc = taskData.data.build break; } - - return sendToLagoonTasks(taskId, { ...taskData, key: updatedKey }); + // send the task to the queue + if (project.organization != null) { + const curOrg = await getOrganizationById(project.organization); + const organization = { + name: curOrg.name, + id: curOrg.id, + } + miscTaskData.project.organization = organization + } + return sendToLagoonTasks(deployTarget+':misc', miscTaskData); } export const consumeTasks = async function( diff --git a/node-packages/commons/src/types.ts b/node-packages/commons/src/types.ts new file mode 100644 index 0000000000..3184773288 --- /dev/null +++ b/node-packages/commons/src/types.ts @@ -0,0 +1,3 @@ +export enum InternalEnvVariableScope { + INTERNAL_SYSTEM = 'internal_system', +} diff --git a/services/api/database/migrations/20240312010000_remove_activesystems.js b/services/api/database/migrations/20240312010000_remove_activesystems.js new file mode 100644 index 0000000000..cc6ff5f63a --- /dev/null +++ b/services/api/database/migrations/20240312010000_remove_activesystems.js @@ -0,0 +1,29 @@ +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.up = function(knex) { + return knex.schema + .alterTable('project', (table) => { + table.dropColumn('active_systems_deploy'); + table.dropColumn('active_systems_promote'); + table.dropColumn('active_systems_remove'); + table.dropColumn('active_systems_task'); + table.dropColumn('active_systems_misc'); + }); +}; + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.down = function(knex) { + return knex.schema + .alterTable('project', (table) => { + table.string('active_systems_deploy', 300); + table.string('active_systems_promote', 300); + table.string('active_systems_remove', 300); + table.string('active_systems_task', 300); + table.string('active_systems_misc', 300); + }); +}; diff --git a/services/api/src/mocks.js b/services/api/src/mocks.js index a7e62257c7..9112901fe0 100644 --- a/services/api/src/mocks.js +++ b/services/api/src/mocks.js @@ -239,10 +239,6 @@ MIIJKQIBAAKCAgEA+o[...]P0yoL8BoQQG2jCvYfWh6vyglQdrDYx/o6/8ecTwXokKKh6fg1q -----END RSA PRIVATE KEY-----`, subfolder: '', notifications: [mocks.Notification()], - activeSystemsDeploy: 'lagoon_controllerBuildDeploy', - activeSystemsPromote: 'lagoon_controllerBuildDeploy', - activeSystemsRemove: 'lagoon_controllerRemove', - activeSystemsTask: 'lagoon_controllerJob', branches: faker.random.arrayElement(['true', 'false', '^(master|staging)$']), pullrequests: faker.random.arrayElement(['true', 'false', '[BUILD]']), productionEnvironment: 'master', diff --git a/services/api/src/resources/project/resolvers.ts b/services/api/src/resources/project/resolvers.ts index 247f184838..8b916ee8ba 100644 --- a/services/api/src/resources/project/resolvers.ts +++ b/services/api/src/resources/project/resolvers.ts @@ -623,11 +623,6 @@ export const updateProject: ResolverFn = async ( privateKey, subfolder, routerPattern, - activeSystemsDeploy, - activeSystemsRemove, - activeSystemsTask, - activeSystemsMisc, - activeSystemsPromote, branches, productionEnvironment, productionRoutes, @@ -809,11 +804,6 @@ export const updateProject: ResolverFn = async ( privateKey, subfolder, routerPattern, - activeSystemsDeploy, - activeSystemsRemove, - activeSystemsTask, - activeSystemsMisc, - activeSystemsPromote, branches, productionEnvironment, productionRoutes, @@ -915,11 +905,6 @@ export const updateProject: ResolverFn = async ( privateKey, subfolder, routerPattern, - activeSystemsDeploy, - activeSystemsRemove, - activeSystemsTask, - activeSystemsMisc, - activeSystemsPromote, branches, productionEnvironment, productionRoutes, diff --git a/services/api/src/resources/project/sql.ts b/services/api/src/resources/project/sql.ts index 0ef7035cf8..5f247989dd 100644 --- a/services/api/src/resources/project/sql.ts +++ b/services/api/src/resources/project/sql.ts @@ -125,11 +125,6 @@ export const Sql = { routerPattern, openshift, openshiftProjectPattern, - activeSystemsDeploy = "lagoon_controllerBuildDeploy", - activeSystemsPromote = "lagoon_controllerBuildDeploy", - activeSystemsRemove = "lagoon_controllerRemove", - activeSystemsTask = "lagoon_controllerJob", - activeSystemsMisc = "lagoon_controllerMisc", branches = "true", pullrequests = "true", productionEnvironment, @@ -159,11 +154,6 @@ export const Sql = { privateKey, subfolder, routerPattern, - activeSystemsDeploy, - activeSystemsPromote, - activeSystemsRemove, - activeSystemsTask, - activeSystemsMisc, branches, productionEnvironment, productionRoutes, diff --git a/services/api/src/typeDefs.js b/services/api/src/typeDefs.js index e3589df654..d5f87d757a 100644 --- a/services/api/src/typeDefs.js +++ b/services/api/src/typeDefs.js @@ -681,31 +681,11 @@ const typeDefs = gql` Notifications that should be sent for this project """ notifications(type: NotificationType, contentType: NotificationContentType, notificationSeverityThreshold: ProblemSeverityRating): [Notification] - """ - Which internal Lagoon System is responsible for deploying - Currently only 'lagoon_controllerBuildDeploy' exists - """ - activeSystemsDeploy: String - """ - Which internal Lagoon System is responsible for promoting - Currently only 'lagoon_controllerBuildDeploy' exists - """ - activeSystemsPromote: String - """ - Which internal Lagoon System is responsible for promoting - Currently only 'lagoon_controllerRemove' exists - """ - activeSystemsRemove: String - """ - Which internal Lagoon System is responsible for tasks - Currently only 'lagoon_controllerJob' exists - """ - activeSystemsTask: String - """ - Which internal Lagoon System is responsible for miscellaneous tasks - Currently only 'lagoon_controllerMisc' exists - """ - activeSystemsMisc: String + activeSystemsDeploy: String @deprecated(reason: "No longer in use") + activeSystemsPromote: String @deprecated(reason: "No longer in use") + activeSystemsRemove: String @deprecated(reason: "No longer in use") + activeSystemsTask: String @deprecated(reason: "No longer in use") + activeSystemsMisc: String @deprecated(reason: "No longer in use") """ Which branches should be deployed, can be one of: - \`true\` - all branches are deployed diff --git a/services/webhooks2tasks/src/handlers/bitbucketBranchDeleted.ts b/services/webhooks2tasks/src/handlers/bitbucketBranchDeleted.ts index 0bbeec9203..35b4004ad6 100644 --- a/services/webhooks2tasks/src/handlers/bitbucketBranchDeleted.ts +++ b/services/webhooks2tasks/src/handlers/bitbucketBranchDeleted.ts @@ -59,8 +59,6 @@ export async function bitbucketBranchDeleted(webhook: WebhookRequestData, projec meta.error switch (error.name) { case "ProjectNotFound": - case "NoActiveSystemsDefined": - case "UnknownActiveSystem": // These are not real errors and also they will happen many times. We just log them locally but not throw an error sendToLagoonLogs('info', project.name, uuid, `${webhooktype}:${event}:handledButNoTask`, meta, `*[${project.name}]* \`${meta.branch}\` deleted. No remove task created, reason: ${error}` diff --git a/services/webhooks2tasks/src/handlers/bitbucketPullRequestClosed.ts b/services/webhooks2tasks/src/handlers/bitbucketPullRequestClosed.ts index 94089e2eb8..2a7d9d2cbc 100644 --- a/services/webhooks2tasks/src/handlers/bitbucketPullRequestClosed.ts +++ b/services/webhooks2tasks/src/handlers/bitbucketPullRequestClosed.ts @@ -61,8 +61,6 @@ export async function bitbucketPullRequestClosed(webhook: WebhookRequestData, pr } catch (error) { switch (error.name) { case "ProjectNotFound": - case "NoActiveSystemsDefined": - case "UnknownActiveSystem": // These are not real errors and also they will happen many times. We just log them locally but not throw an error sendToLagoonLogs('info', project.name, uuid, `${webhooktype}:${event}:handledButNoTask`, meta, `*[${project.name}]* \`${body.pullrequest.id}\` deleted. No remove task created, reason: ${error}` diff --git a/services/webhooks2tasks/src/handlers/bitbucketPullRequestUpdated.ts b/services/webhooks2tasks/src/handlers/bitbucketPullRequestUpdated.ts index 66b1957b8a..5ca5c901ab 100644 --- a/services/webhooks2tasks/src/handlers/bitbucketPullRequestUpdated.ts +++ b/services/webhooks2tasks/src/handlers/bitbucketPullRequestUpdated.ts @@ -80,8 +80,6 @@ export async function bitbucketPullRequestUpdated(webhook: WebhookRequestData, p } catch (error) { switch (error.name) { case "ProjectNotFound": - case "NoActiveSystemsDefined": - case "UnknownActiveSystem": // These are not real errors and also they will happen many times. We just log them locally but not throw an error sendToLagoonLogs('info', project.name, uuid, `${webhooktype}:${event}:handledButNoTask`, meta, `*[${project.name}]* PR ${body.object_attributes.id} updated. No deploy task created, reason: ${error}` diff --git a/services/webhooks2tasks/src/handlers/bitbucketPush.ts b/services/webhooks2tasks/src/handlers/bitbucketPush.ts index 451dc56e80..25c9e18e48 100644 --- a/services/webhooks2tasks/src/handlers/bitbucketPush.ts +++ b/services/webhooks2tasks/src/handlers/bitbucketPush.ts @@ -80,8 +80,6 @@ export async function bitbucketPush(webhook: WebhookRequestData, project: Projec } catch (error) { switch (error.name) { case "ProjectNotFound": - case "NoActiveSystemsDefined": - case "UnknownActiveSystem": case "NoNeedToDeployBranch": // These are not real errors and also they will happen many times. We just log them locally but not throw an error sendToLagoonLogs('info', project.name, uuid, `${webhooktype}:${event}:handledButNoTask`, meta, diff --git a/services/webhooks2tasks/src/handlers/giteaBranchDeleted.ts b/services/webhooks2tasks/src/handlers/giteaBranchDeleted.ts index beaebdae13..84f9d3fdef 100644 --- a/services/webhooks2tasks/src/handlers/giteaBranchDeleted.ts +++ b/services/webhooks2tasks/src/handlers/giteaBranchDeleted.ts @@ -57,8 +57,6 @@ export async function giteaBranchDeleted(webhook: WebhookRequestData, project: P meta.error = error switch (error.name) { case "ProjectNotFound": - case "NoActiveSystemsDefined": - case "UnknownActiveSystem": // These are not real errors and also they will happen many times. We just log them locally but not throw an error sendToLagoonLogs('info', project.name, uuid, `${webhooktype}:${event}:handledButNoTask`, meta, `*[${project.name}]* \`${meta.branch}\` deleted. No remove task created, reason: ${error}` diff --git a/services/webhooks2tasks/src/handlers/giteaPullRequestClosed.ts b/services/webhooks2tasks/src/handlers/giteaPullRequestClosed.ts index 6894919760..e665259283 100644 --- a/services/webhooks2tasks/src/handlers/giteaPullRequestClosed.ts +++ b/services/webhooks2tasks/src/handlers/giteaPullRequestClosed.ts @@ -64,8 +64,6 @@ export async function giteaPullRequestClosed(webhook: WebhookRequestData, projec meta.error = error switch (error.name) { case "ProjectNotFound": - case "NoActiveSystemsDefined": - case "UnknownActiveSystem": // These are not real errors and also they will happen many times. We just log them locally but not throw an error sendToLagoonLogs('info', project.name, uuid, `${webhooktype}:${event}:handledButNoTask`, meta, `*[${project.name}]* PR ${body.number} closed. No remove task created, reason: ${error}` diff --git a/services/webhooks2tasks/src/handlers/giteaPullRequestOpened.ts b/services/webhooks2tasks/src/handlers/giteaPullRequestOpened.ts index f2559771d5..db53277d9d 100644 --- a/services/webhooks2tasks/src/handlers/giteaPullRequestOpened.ts +++ b/services/webhooks2tasks/src/handlers/giteaPullRequestOpened.ts @@ -81,8 +81,6 @@ export async function giteaPullRequestOpened(webhook: WebhookRequestData, projec } catch (error) { switch (error.name) { case "ProjectNotFound": - case "NoActiveSystemsDefined": - case "UnknownActiveSystem": // These are not real errors and also they will happen many times. We just log them locally but not throw an error sendToLagoonLogs('info', project.name, uuid, `${webhooktype}:${event}:handledButNoTask`, meta, `*[${project.name}]* PR ${body.number} opened. No deploy task created, reason: ${error}` diff --git a/services/webhooks2tasks/src/handlers/giteaPullRequestSynchronize.ts b/services/webhooks2tasks/src/handlers/giteaPullRequestSynchronize.ts index aca515674e..63f7cefffe 100644 --- a/services/webhooks2tasks/src/handlers/giteaPullRequestSynchronize.ts +++ b/services/webhooks2tasks/src/handlers/giteaPullRequestSynchronize.ts @@ -99,8 +99,6 @@ export async function giteaPullRequestSynchronize(webhook: WebhookRequestData, p } catch (error) { switch (error.name) { case "ProjectNotFound": - case "NoActiveSystemsDefined": - case "UnknownActiveSystem": // These are not real errors and also they will happen many times. We just log them locally but not throw an error sendToLagoonLogs('info', project.name, uuid, `${webhooktype}:${event}:handledButNoTask`, meta, `*[${project.name}]* PR ${body.number} opened. No deploy task created, reason: ${error}` diff --git a/services/webhooks2tasks/src/handlers/giteaPush.ts b/services/webhooks2tasks/src/handlers/giteaPush.ts index bbe0390891..5a91e6d29b 100644 --- a/services/webhooks2tasks/src/handlers/giteaPush.ts +++ b/services/webhooks2tasks/src/handlers/giteaPush.ts @@ -81,8 +81,6 @@ export async function giteaPush(webhook: WebhookRequestData, project: Project) { } catch (error) { switch (error.name) { case "ProjectNotFound": - case "NoActiveSystemsDefined": - case "UnknownActiveSystem": case "NoNeedToDeployBranch": // These are not real errors and also they will happen many times. We just log them locally but not throw an error sendToLagoonLogs('info', project.name, uuid, `${webhooktype}:${event}:handledButNoTask`, meta, diff --git a/services/webhooks2tasks/src/handlers/githubBranchDeleted.ts b/services/webhooks2tasks/src/handlers/githubBranchDeleted.ts index b9f42c6f39..b7a7b72710 100644 --- a/services/webhooks2tasks/src/handlers/githubBranchDeleted.ts +++ b/services/webhooks2tasks/src/handlers/githubBranchDeleted.ts @@ -57,8 +57,6 @@ export async function githubBranchDeleted(webhook: WebhookRequestData, project: meta.error = error switch (error.name) { case "ProjectNotFound": - case "NoActiveSystemsDefined": - case "UnknownActiveSystem": // These are not real errors and also they will happen many times. We just log them locally but not throw an error sendToLagoonLogs('info', project.name, uuid, `${webhooktype}:${event}:handledButNoTask`, meta, `*[${project.name}]* \`${meta.branch}\` deleted. No remove task created, reason: ${error}` diff --git a/services/webhooks2tasks/src/handlers/githubPullRequestClosed.ts b/services/webhooks2tasks/src/handlers/githubPullRequestClosed.ts index 445825f20d..813dd147b3 100644 --- a/services/webhooks2tasks/src/handlers/githubPullRequestClosed.ts +++ b/services/webhooks2tasks/src/handlers/githubPullRequestClosed.ts @@ -64,8 +64,6 @@ export async function githubPullRequestClosed(webhook: WebhookRequestData, proje meta.error = error switch (error.name) { case "ProjectNotFound": - case "NoActiveSystemsDefined": - case "UnknownActiveSystem": // These are not real errors and also they will happen many times. We just log them locally but not throw an error sendToLagoonLogs('info', project.name, uuid, `${webhooktype}:${event}:handledButNoTask`, meta, `*[${project.name}]* PR ${body.number} closed. No remove task created, reason: ${error}` diff --git a/services/webhooks2tasks/src/handlers/githubPullRequestOpened.ts b/services/webhooks2tasks/src/handlers/githubPullRequestOpened.ts index d110fbe73d..3e1d962d5f 100644 --- a/services/webhooks2tasks/src/handlers/githubPullRequestOpened.ts +++ b/services/webhooks2tasks/src/handlers/githubPullRequestOpened.ts @@ -81,8 +81,6 @@ export async function githubPullRequestOpened(webhook: WebhookRequestData, proje } catch (error) { switch (error.name) { case "ProjectNotFound": - case "NoActiveSystemsDefined": - case "UnknownActiveSystem": // These are not real errors and also they will happen many times. We just log them locally but not throw an error sendToLagoonLogs('info', project.name, uuid, `${webhooktype}:${event}:handledButNoTask`, meta, `*[${project.name}]* PR ${body.number} opened. No deploy task created, reason: ${error}` diff --git a/services/webhooks2tasks/src/handlers/githubPullRequestSynchronize.ts b/services/webhooks2tasks/src/handlers/githubPullRequestSynchronize.ts index 5223e44013..3990d4e900 100644 --- a/services/webhooks2tasks/src/handlers/githubPullRequestSynchronize.ts +++ b/services/webhooks2tasks/src/handlers/githubPullRequestSynchronize.ts @@ -99,8 +99,6 @@ export async function githubPullRequestSynchronize(webhook: WebhookRequestData, } catch (error) { switch (error.name) { case "ProjectNotFound": - case "NoActiveSystemsDefined": - case "UnknownActiveSystem": // These are not real errors and also they will happen many times. We just log them locally but not throw an error sendToLagoonLogs('info', project.name, uuid, `${webhooktype}:${event}:handledButNoTask`, meta, `*[${project.name}]* PR ${body.number} opened. No deploy task created, reason: ${error}` diff --git a/services/webhooks2tasks/src/handlers/githubPush.ts b/services/webhooks2tasks/src/handlers/githubPush.ts index c186ff61bf..b23522805e 100644 --- a/services/webhooks2tasks/src/handlers/githubPush.ts +++ b/services/webhooks2tasks/src/handlers/githubPush.ts @@ -80,8 +80,6 @@ export async function githubPush(webhook: WebhookRequestData, project: Project) } catch (error) { switch (error.name) { case "ProjectNotFound": - case "NoActiveSystemsDefined": - case "UnknownActiveSystem": case "NoNeedToDeployBranch": // These are not real errors and also they will happen many times. We just log them locally but not throw an error sendToLagoonLogs('info', project.name, uuid, `${webhooktype}:${event}:handledButNoTask`, meta, diff --git a/services/webhooks2tasks/src/handlers/gitlabBranchDeleted.ts b/services/webhooks2tasks/src/handlers/gitlabBranchDeleted.ts index e153414320..96c31477c8 100644 --- a/services/webhooks2tasks/src/handlers/gitlabBranchDeleted.ts +++ b/services/webhooks2tasks/src/handlers/gitlabBranchDeleted.ts @@ -60,8 +60,6 @@ export async function gitlabBranchDeleted(webhook: WebhookRequestData, project: meta.error = error switch (error.name) { case "ProjectNotFound": - case "NoActiveSystemsDefined": - case "UnknownActiveSystem": // These are not real errors and also they will happen many times. We just log them locally but not throw an error sendToLagoonLogs('info', project.name, uuid, `${webhooktype}:remove:handledButNoTask`, meta, `*[${project.name}]* \`${meta.branch}\` deleted. No remove task created, reason: ${error}` diff --git a/services/webhooks2tasks/src/handlers/gitlabPullRequestClosed.ts b/services/webhooks2tasks/src/handlers/gitlabPullRequestClosed.ts index 53b9f6f615..1f3fa0fff5 100644 --- a/services/webhooks2tasks/src/handlers/gitlabPullRequestClosed.ts +++ b/services/webhooks2tasks/src/handlers/gitlabPullRequestClosed.ts @@ -61,8 +61,6 @@ export async function gitlabPullRequestClosed(webhook: WebhookRequestData, proje } catch (error) { switch (error.name) { case "ProjectNotFound": - case "NoActiveSystemsDefined": - case "UnknownActiveSystem": // These are not real errors and also they will happen many times. We just log them locally but not throw an error sendToLagoonLogs('info', project.name, uuid, `${webhooktype}:${event}:handledButNoTask`, meta, `*[${project.name}]* PR ${body.object_attributes.iid} closed. No remove task created, reason: ${error}` diff --git a/services/webhooks2tasks/src/handlers/gitlabPullRequestOpened.ts b/services/webhooks2tasks/src/handlers/gitlabPullRequestOpened.ts index 97fc822df7..9064da0aeb 100644 --- a/services/webhooks2tasks/src/handlers/gitlabPullRequestOpened.ts +++ b/services/webhooks2tasks/src/handlers/gitlabPullRequestOpened.ts @@ -81,8 +81,6 @@ export async function gitlabPullRequestOpened(webhook: WebhookRequestData, proje } catch (error) { switch (error.name) { case "ProjectNotFound": - case "NoActiveSystemsDefined": - case "UnknownActiveSystem": // These are not real errors and also they will happen many times. We just log them locally but not throw an error sendToLagoonLogs('info', project.name, uuid, `${webhooktype}:${event}:handledButNoTask`, meta, `*[${project.name}]* PR ${body.object_attributes.iid} opened. No deploy task created, reason: ${error}` diff --git a/services/webhooks2tasks/src/handlers/gitlabPullRequestUpdated.ts b/services/webhooks2tasks/src/handlers/gitlabPullRequestUpdated.ts index 7b7004c181..05be8daeda 100644 --- a/services/webhooks2tasks/src/handlers/gitlabPullRequestUpdated.ts +++ b/services/webhooks2tasks/src/handlers/gitlabPullRequestUpdated.ts @@ -82,8 +82,6 @@ export async function gitlabPullRequestUpdated(webhook: WebhookRequestData, proj } catch (error) { switch (error.name) { case "ProjectNotFound": - case "NoActiveSystemsDefined": - case "UnknownActiveSystem": // These are not real errors and also they will happen many times. We just log them locally but not throw an error sendToLagoonLogs('info', project.name, uuid, `${webhooktype}:${event}:handledButNoTask`, meta, `*[${project.name}]* PR ${body.object_attributes.iid} updated. No deploy task created, reason: ${error}` diff --git a/services/webhooks2tasks/src/handlers/gitlabPush.ts b/services/webhooks2tasks/src/handlers/gitlabPush.ts index a89fa3452f..d24648a9dd 100644 --- a/services/webhooks2tasks/src/handlers/gitlabPush.ts +++ b/services/webhooks2tasks/src/handlers/gitlabPush.ts @@ -81,8 +81,6 @@ export async function gitlabPush(webhook: WebhookRequestData, project: Project) } catch (error) { switch (error.name) { case "ProjectNotFound": - case "NoActiveSystemsDefined": - case "UnknownActiveSystem": case "NoNeedToDeployBranch": // These are not real errors and also they will happen many times. We just log them locally but not throw an error sendToLagoonLogs('info', project.name, uuid, `${webhooktype}:${event}:handledButNoTask`, meta, diff --git a/services/webhooks2tasks/src/types.ts b/services/webhooks2tasks/src/types.ts index 98bd8eaaa3..e256118da9 100644 --- a/services/webhooks2tasks/src/types.ts +++ b/services/webhooks2tasks/src/types.ts @@ -1,55 +1,55 @@ +import { Project as LagoonProject } from '@lagoon/commons/dist/api'; + /** * Webhook related types for incoming Events */ export interface removeData { - projectName: string, - pullrequestNumber?: number, - pullrequestTitle?: string, - openshiftProjectName?: string, - forceDeleteProductionEnvironment?: boolean, - branchName?: string, - branch?: string, - pullrequest?: string, - type: string -}; + branch?: string; + branchName?: string; + forceDeleteProductionEnvironment?: boolean; + openshiftProjectName?: string; + projectName: string; + pullrequest?: string; + pullrequestNumber?: number; + pullrequestTitle?: string; + type: string; +} export interface deployData { - projectName: string, - branchName: string, - sha?: string, - repoUrl?: string, - repoName?: string, - pullrequestTitle?: string, - pullrequestNumber?: number, - pullrequestUrl?: string, - type?: string, - headBranchName?: string, - headSha?: string, - baseBranchName?: string, - baseSha?: string, - buildName?: string, - buildPriority?: string, - bulkId?: string, - buildVariables?: any, - sourceUser?: string, + baseBranchName?: string; + baseSha?: string; + branchName: string; + buildName?: string; + buildPriority?: string; + buildVariables?: any; + bulkId?: string; + headBranchName?: string; + headSha?: string; + projectName: string; + pullrequestNumber?: number; + pullrequestTitle?: string; + pullrequestUrl?: string; + repoName?: string; + repoUrl?: string; + sha?: string; sourceType?: string, -}; + sourceUser?: string, + type?: string; +} export interface WebhookRequestData { - webhooktype: string, - event: string, - giturl: string, - uuid?: string, - body?: any, - user?: any, - sender?: any -}; + body?: any; + event: string; + giturl: string; + sender?: any; + user?: any; + uuid?: string; + webhooktype: string; +} -export interface Project { - slack: any, - name: string, - deploymentsDisabled: number, - openshift: any, - productionEnvironment?: string -}; +export type Project = Pick< + LagoonProject, + | 'name' + | 'deploymentsDisabled' +>; diff --git a/services/webhooks2tasks/src/webhooks/projects.ts b/services/webhooks2tasks/src/webhooks/projects.ts index a8cf5e3ee2..959cfc0d0f 100644 --- a/services/webhooks2tasks/src/webhooks/projects.ts +++ b/services/webhooks2tasks/src/webhooks/projects.ts @@ -1,7 +1,7 @@ import { ChannelWrapper } from 'amqp-connection-manager'; import { ConsumeMessage } from 'amqplib'; import { logger } from '@lagoon/commons/dist/logs/local-logger'; -import { getProjectsByGitUrl } from '@lagoon/commons/dist/api'; +import { Project as LagoonProject, ProjectNotFound, lagoonApi } from '@lagoon/commons/dist/api'; import { sendToLagoonLogs } from '@lagoon/commons/dist/logs/lagoon-logger'; import { githubPullRequestClosed } from '../handlers/githubPullRequestClosed'; import { githubPullRequestOpened } from '../handlers/githubPullRequestOpened'; @@ -33,13 +33,31 @@ export async function processProjects( channelWrapperWebhooks: ChannelWrapper ): Promise { const webhook: WebhookRequestData = JSON.parse(rabbitMsg.content.toString()); - - let projects: Project[]; - const { webhooktype, event, giturl, uuid, body } = webhook; try { - projects = await getProjectsByGitUrl(giturl); + const result = await lagoonApi.query( + ` + query webhookProcessProjects($giturl: String!) { + allProjects(gitUrl: $giturl) { + name + deploymentsDisabled + } + } + `, + { giturl } + ) as { + allProjects: Pick< + LagoonProject, + 'name' | 'deploymentsDisabled' + >[]; + } + + if (!result || !result.allProjects || !result.allProjects.length) { + throw new ProjectNotFound(`Cannot find project for git repo ${giturl}`); + } + + var projects = result.allProjects; } catch (error) { if (error.name == 'ProjectNotFound') { const meta = { From 945437c0d657917473fe7ba5040022cea31830d6 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Tue, 12 Mar 2024 21:49:26 -0500 Subject: [PATCH 2/2] Removing dead code --- node-packages/commons/src/api.ts | 351 ------------------ node-packages/commons/src/util/func.ts | 1 - .../src/bitbucket-sync/repo-permissions.ts | 18 +- services/api/src/gitlab-sync/groups.ts | 15 +- services/api/src/gitlab-sync/projects.ts | 19 +- services/api/src/gitlab-sync/users.ts | 6 +- .../api/src/helpers/reset-project-keys.ts | 7 +- services/api/src/models/group.ts | 5 - services/api/src/models/user.ts | 4 +- .../api/src/resources/deployment/resolvers.ts | 14 - .../src/resources/env-variables/resolvers.ts | 4 +- .../src/resources/environment/resolvers.ts | 5 +- services/api/src/resources/fact/resolvers.ts | 3 - services/api/src/resources/group/resolvers.ts | 2 - .../api/src/resources/organization/helpers.ts | 4 +- .../src/resources/organization/resolvers.ts | 4 +- .../api/src/resources/project/resolvers.ts | 3 - services/api/src/resources/user/resolvers.ts | 1 - services/api/src/util/auth.ts | 5 - services/api/src/util/func.ts | 1 - .../problems/processDrutinyResults.ts | 8 +- 21 files changed, 49 insertions(+), 431 deletions(-) diff --git a/node-packages/commons/src/api.ts b/node-packages/commons/src/api.ts index 27cbbaad01..05c1a20848 100644 --- a/node-packages/commons/src/api.ts +++ b/node-packages/commons/src/api.ts @@ -684,159 +684,6 @@ export const allProjectsInGroup = (groupInput: { } ); -export async function getMicrosoftTeamsInfoForProject( - project: string, contentType = 'DEPLOYMENT' -): Promise { - const notificationsFragment = graphqlapi.createFragment(` - fragment on NotificationMicrosoftTeams { - webhook - contentType - notificationSeverityThreshold - } - `); - - const result = await graphqlapi.query(` - { - project:projectByName(name: "${project}") { - microsoftTeams: notifications(type: MICROSOFTTEAMS, contentType: ${contentType}) { - ...${notificationsFragment} - } - } - } - `); - - if (!result || !result.project || !result.project.microsoftTeams) { - throw new ProjectNotFound( - `Cannot find Microsoft Teams information for project ${project}` - ); - } - - return result.project.microsoftTeams; -} - -export async function getRocketChatInfoForProject( - project: string, contentType = 'DEPLOYMENT' -): Promise { - const notificationsFragment = graphqlapi.createFragment(` - fragment on NotificationRocketChat { - webhook - channel - contentType - notificationSeverityThreshold - } - `); - - const result = await graphqlapi.query(` - { - project:projectByName(name: "${project}") { - rocketchats: notifications(type: ROCKETCHAT, contentType: ${contentType}) { - ...${notificationsFragment} - } - } - } - `); - - if (!result || !result.project || !result.project.rocketchats) { - throw new ProjectNotFound( - `Cannot find rocketchat information for project ${project}` - ); - } - - return result.project.rocketchats; -} - -export async function getSlackinfoForProject( - project: string, contentType = 'DEPLOYMENT' -): Promise { - const notificationsFragment = graphqlapi.createFragment(` - fragment on NotificationSlack { - webhook - channel - contentType - notificationSeverityThreshold - } - `); - - const result = await graphqlapi.query(` - { - project:projectByName(name: "${project}") { - slacks: notifications(type: SLACK, contentType: ${contentType}) { - ...${notificationsFragment} - } - } - } - `); - - if (!result || !result.project || !result.project.slacks) { - throw new ProjectNotFound( - `Cannot find slack information for project ${project}` - ); - } - - return result.project.slacks; -} - -export async function getWebhookNotificationInfoForProject( - project: string, contentType = 'DEPLOYMENT' -): Promise { - const notificationsFragment = graphqlapi.createFragment(` - fragment on NotificationWebhook { - webhook - contentType - notificationSeverityThreshold - } - `); - - const result = await graphqlapi.query(` - { - project:projectByName(name: "${project}") { - webhook: notifications(type: WEBHOOK, contentType: ${contentType}) { - ...${notificationsFragment} - } - } - } - `); - - if (!result || !result.project || !result.project.webhook) { - throw new ProjectNotFound( - `Cannot find Webhook Notification information for project ${project}` - ); - } - - return result.project.webhook; -} - - -export async function getEmailInfoForProject( - project: string, contentType = 'DEPLOYMENT' -): Promise { - const notificationsFragment = graphqlapi.createFragment(` - fragment on NotificationEmail { - emailAddress - contentType - notificationSeverityThreshold - } - `); - - const result = await graphqlapi.query(` - { - project:projectByName(name: "${project}") { - emails: notifications(type: EMAIL, contentType: ${contentType}) { - ...${notificationsFragment} - } - } - } - `); - - if (!result || !result.project || !result.project.emails) { - throw new ProjectNotFound( - `Cannot find email information for project ${project}` - ); - } - - return result.project.emails; -} - export async function getEnvironmentByName( name: string, projectId: number, @@ -869,37 +716,6 @@ export async function getEnvironmentByName( return result; } - -export async function getEnvironmentById( - id: number -): Promise { - const result = await graphqlapi.query(` - { - environmentById(id: ${id}) { - id, - name, - route, - routes, - deployType, - autoIdle, - environmentType, - openshiftProjectName, - updated, - created, - deleted, - } - } - `); - - if (!result || !result.environmentById) { - throw new EnvironmentNotFound( - `Cannot find environment for id ${id}\n${result.environmentById}` - ); - } - - return result; -} - export async function getEnvironmentByIdWithVariables( id: number ): Promise { @@ -934,40 +750,6 @@ export async function getEnvironmentByIdWithVariables( return result; } -export async function getDeploymentByName( - openshiftProjectName: string, - deploymentName: string, -): Promise { - const result = await graphqlapi.query(` - { - environment:environmentByOpenshiftProjectName( openshiftProjectName: "${openshiftProjectName}") { - id - name - openshiftProjectName - project { - id - name - } - deployments(name: "${deploymentName}") { - id - name - uiLink - } - } - } - `); - - if (!result || !result.environment) { - throw new EnvironmentNotFound( - `Cannot find deployment ${deploymentName} by projectName ${openshiftProjectName}\n${ - result.environment - }`, - ); - } - - return result; -} - export async function getEnvironmentByOpenshiftProjectName( openshiftProjectName: string ): Promise { @@ -1051,45 +833,6 @@ export const addOrUpdateEnvironment = ( } ); -export const updateEnvironment = ( - environmentId: number, - patch: string -): Promise => - graphqlapi.query(` - mutation { - updateEnvironment(input: { - id: ${environmentId}, - patch: ${patch} - }) { - id - name - } - } - `); - -export async function deleteEnvironment( - name: string, - project: string, - execute: boolean = true -): Promise { - return graphqlapi.mutate( - ` - ($name: String!, $project: String!, $execute: Boolean) { - deleteEnvironment(input: { - name: $name - project: $project - execute: $execute - }) - } - `, - { - name, - project, - execute - } - ); -} - interface GetOpenshiftInfoForProjectResult { project: Pick< Project, @@ -1300,25 +1043,6 @@ export async function getOrganizationById(id: number): Promise { return result.organization; } -export const setEnvironmentServices = ( - environment: number, - services: string[] -): Promise => - graphqlapi.mutate( - ` - ($environment: Int!, $services: [String]!) { - setEnvironmentServices(input: { - environment: $environment - services: $services - }) { - id - name - } - } - `, - { environment, services } - ); - const deploymentFragment = graphqlapi.createFragment(` fragment on Deployment { id @@ -1358,18 +1082,6 @@ fragment on Openshift { } `); -export const getDeploymentByRemoteId = (id: string): Promise => - graphqlapi.query( - ` - query deploymentByRemoteId($id: String!) { - deploymentByRemoteId(id: $id) { - ...${deploymentFragment} - } - } -`, - { id } - ); - export const addDeployment = ( name: string, status: string, @@ -1482,24 +1194,6 @@ export const addDeployment = ( }, ); -export const updateDeployment = ( - id: number, - patch: DeploymentPatch -): Promise => - graphqlapi.mutate( - ` - ($id: Int!, $patch: UpdateDeploymentPatchInput!) { - updateDeployment(input: { - id: $id - patch: $patch - }) { - ...${deploymentFragment} - } - } -`, - { id, patch } - ); - const taskFragment = graphqlapi.createFragment(` fragment on Task { id @@ -1515,21 +1209,6 @@ fragment on Task { } `); -export const updateTask = (id: number, patch: TaskPatch): Promise => - graphqlapi.mutate( - ` - ($id: Int!, $patch: UpdateTaskPatchInput!) { - updateTask(input: { - id: $id - patch: $patch - }) { - ...${taskFragment} - } - } -`, - { id, patch } - ); - export const sanitizeGroupName = pipe( replace(/[^a-zA-Z0-9-]/g, '-'), toLower @@ -1721,33 +1400,3 @@ export const getProblemHarborScanMatches = () => graphqlapi.query( } }` ); - -const bulkDeploymentFragment = graphqlapi.createFragment(` -fragment on Deployment { - id - name - status - created - started - completed - remoteId - uiLink - environment { - name - } - priority - bulkId -} -`); - -export const getDeploymentsByBulkId = (id: string): Promise => - graphqlapi.query( - ` - query deploymentsByBulkId($id: String!) { - deploymentsByBulkId(id: $id) { - ...${bulkDeploymentFragment} - } - } -`, - { id } - ); diff --git a/node-packages/commons/src/util/func.ts b/node-packages/commons/src/util/func.ts index 9b8edd606e..dc25b5b320 100644 --- a/node-packages/commons/src/util/func.ts +++ b/node-packages/commons/src/util/func.ts @@ -1,4 +1,3 @@ -// @ts-ignore import { unless, is, isNil, isEmpty, partialRight, complement } from 'ramda'; export const isNumber = is(Number); diff --git a/services/api/src/bitbucket-sync/repo-permissions.ts b/services/api/src/bitbucket-sync/repo-permissions.ts index 0813ff1009..8808892c69 100644 --- a/services/api/src/bitbucket-sync/repo-permissions.ts +++ b/services/api/src/bitbucket-sync/repo-permissions.ts @@ -1,6 +1,12 @@ import * as R from 'ramda'; import * as bitbucketApi from '@lagoon/commons/dist/bitbucketApi'; -import * as api from '@lagoon/commons/dist/api'; +import { + addUserToGroup, + removeUserFromGroup, + getGroupMembersByGroupName, + getProjectsByGroupName, + addUser as apiAddUser +} from '@lagoon/commons/dist/api'; import { logger } from '@lagoon/commons/dist/logs/local-logger'; import redis, { ClientOpts } from 'redis'; import { promisify } from 'util'; @@ -46,7 +52,7 @@ const isNotEmpty = R.complement(R.isEmpty); // Returns false if adding user failed and no user exists. const addUser = async (email: string): Promise => { try { - await api.addUser(email); + await apiAddUser(email); } catch (err) { if (!userExistsTest(err.message)) { logger.error( @@ -174,7 +180,7 @@ const syncUsersForProjects = async (redis, projects) => { //Now we check if we need to change this users' permissions if(getUsersCurrentPermission(lagoonUsersInGroupTotal, email) != BitbucketPermsToLagoonPerms[bbPerm]) { - await api.addUserToGroup( + await addUserToGroup( email, lagoonProjectGroup, BitbucketPermsToLagoonPerms[bbPerm] @@ -209,7 +215,7 @@ const syncUsersForProjects = async (redis, projects) => { for (const user of deleteUsers) { try { - await api.removeUserFromGroup(user, lagoonProjectGroup); + await removeUserFromGroup(user, lagoonProjectGroup); } catch (err) { logger.error( `Could not remove user (${user}) from group (${lagoonProjectGroup}): ${err.message}` @@ -248,7 +254,7 @@ function getUsersCurrentPermission(lagoonUsers, email) { } async function getLagoonUsersForGroup(lagoonProjectGroup: string) { - const currentMembersQuery = await api.getGroupMembersByGroupName(lagoonProjectGroup); + const currentMembersQuery = await getGroupMembersByGroupName(lagoonProjectGroup); const lagoonUsers = R.pipe(R.pathOr([], ['groupByName','members']))(currentMembersQuery); return lagoonUsers; } @@ -294,7 +300,7 @@ const getRedisClient = () => { // @ts-ignore redisObj.redisClient.on("ready", async function() { // Get all bitbucket related lagoon projects - const groupQuery = await api.getProjectsByGroupName(LAGOON_SYNC_GROUP); + const groupQuery = await getProjectsByGroupName(LAGOON_SYNC_GROUP); const projects = R.pathOr([], ['groupByName', 'projects'], groupQuery) as [ object ]; diff --git a/services/api/src/gitlab-sync/groups.ts b/services/api/src/gitlab-sync/groups.ts index 27100d2e86..e90e65845e 100644 --- a/services/api/src/gitlab-sync/groups.ts +++ b/services/api/src/gitlab-sync/groups.ts @@ -1,6 +1,11 @@ import * as R from 'ramda'; import * as gitlabApi from '@lagoon/commons/dist/gitlab/api'; -import * as api from '@lagoon/commons/dist/api'; +import { + sanitizeGroupName, + addGroupWithParent, + addGroup, + addUserToGroup +} from '@lagoon/commons/dist/api'; import { logger } from '@lagoon/commons/dist/logs/local-logger'; interface GitlabGroup { @@ -21,15 +26,15 @@ const convertRoleNumberToString = R.cond([ ]); const syncGroup = async group => { - const groupName = api.sanitizeGroupName(group.full_path); + const groupName = sanitizeGroupName(group.full_path); logger.debug(`Processing ${group.name} (${groupName})`); try { if (group.parent_id) { const parentGroup = await gitlabApi.getGroup(group.parent_id); - await api.addGroupWithParent(groupName, api.sanitizeGroupName(parentGroup.full_path)); + await addGroupWithParent(groupName, sanitizeGroupName(parentGroup.full_path)); } else { - await api.addGroup(groupName); + await addGroup(groupName); } } catch (err) { if (!R.test(groupExistsRegex, err.message)) { @@ -42,7 +47,7 @@ const syncGroup = async group => { for (const member of groupMembers) { const user = await gitlabApi.getUser(member.id); - await api.addUserToGroup(user.email, groupName, convertRoleNumberToString(member.access_level)); + await addUserToGroup(user.email, groupName, convertRoleNumberToString(member.access_level)); } }; diff --git a/services/api/src/gitlab-sync/projects.ts b/services/api/src/gitlab-sync/projects.ts index 1d1c3ce281..5d4c8647b9 100644 --- a/services/api/src/gitlab-sync/projects.ts +++ b/services/api/src/gitlab-sync/projects.ts @@ -1,7 +1,14 @@ import * as R from 'ramda'; import * as sshpk from 'sshpk'; import * as gitlabApi from '@lagoon/commons/dist/gitlab/api'; -import * as api from '@lagoon/commons/dist/api'; +import { + sanitizeGroupName, + sanitizeProjectName, + addProject, + getProjectByName, + addGroupToProject, + addUserToGroup +} from '@lagoon/commons/dist/api'; import { logger } from '@lagoon/commons/dist/logs/local-logger'; interface GitlabProject { @@ -26,7 +33,7 @@ const convertRoleNumberToString = R.cond([ const syncProject = async (project) => { const { id, path, ssh_url_to_repo: gitUrl, namespace } = project; - const projectName = api.sanitizeProjectName(path); + const projectName = sanitizeProjectName(path); const openshift = 1; const productionenvironment = "master"; logger.debug(`Processing ${projectName}`); @@ -38,11 +45,11 @@ const syncProject = async (project) => { let lagoonProject; try { - const result = await api.addProject(projectName, gitUrl, openshift, productionenvironment); + const result = await addProject(projectName, gitUrl, openshift, productionenvironment); lagoonProject = R.prop('addProject', result); } catch (err) { if (R.test(projectExistsRegex, err.message)) { - lagoonProject = await api.getProjectByName(projectName); + lagoonProject = await getProjectByName(projectName); } else { throw new Error(`Could not sync (add) gitlab project ${projectName}: ${err.message}`); } @@ -65,14 +72,14 @@ const syncProject = async (project) => { // In Gitlab each project has an Owner, which is in this case a Group that already should be created before. // We add this owner Group to the Project. - await api.addGroupToProject(projectName, api.sanitizeGroupName(namespace.full_path)); + await addGroupToProject(projectName, sanitizeGroupName(namespace.full_path)); const projectMembers = await gitlabApi.getProjectMembers(project.id); for (const member of projectMembers) { const user = await gitlabApi.getUser(member.id); - await api.addUserToGroup(user.email, `project-${projectName}`, convertRoleNumberToString(member.access_level)); + await addUserToGroup(user.email, `project-${projectName}`, convertRoleNumberToString(member.access_level)); } }; diff --git a/services/api/src/gitlab-sync/users.ts b/services/api/src/gitlab-sync/users.ts index de527c8159..0d59f3a862 100644 --- a/services/api/src/gitlab-sync/users.ts +++ b/services/api/src/gitlab-sync/users.ts @@ -1,6 +1,6 @@ import * as R from 'ramda'; import * as gitlabApi from '@lagoon/commons/dist/gitlab/api'; -import * as api from '@lagoon/commons/dist/api'; +import { addUser, updateUser } from '@lagoon/commons/dist/api'; import { logger } from '@lagoon/commons/dist/logs/local-logger'; const usernameExistsRegex = /Username.*?exists/; @@ -22,7 +22,7 @@ const usernameExistsRegex = /Username.*?exists/; } try { - await api.addUser( + await addUser( user.email, firstName, lastName, @@ -34,7 +34,7 @@ const usernameExistsRegex = /Username.*?exists/; logger.error(`Could not sync (add) gitlab user ${user.email} id ${user.id}: ${err.message}`); } else { try { - await api.updateUser( + await updateUser( user.email, { email: user.email, diff --git a/services/api/src/helpers/reset-project-keys.ts b/services/api/src/helpers/reset-project-keys.ts index e174786c09..98daccae65 100644 --- a/services/api/src/helpers/reset-project-keys.ts +++ b/services/api/src/helpers/reset-project-keys.ts @@ -1,7 +1,7 @@ import * as R from 'ramda'; import { parsePrivateKey } from 'sshpk'; import { logger } from '@lagoon/commons/dist/logs/local-logger'; -import * as api from '@lagoon/commons/dist/api'; +import { sanitizeGroupName } from '@lagoon/commons/dist/api'; import * as gitlabApi from '@lagoon/commons/dist/gitlab/api'; import { getKeycloakAdminClient } from '../clients/keycloak-admin'; import { sqlClientPool } from '../clients/sqlClient'; @@ -65,7 +65,7 @@ const generatePrivateKeyEd25519 = R.partial(generatePrivateKey, ['ed25519']); const gitlabProject = R.find( (findProject: GitlabProject) => - api.sanitizeGroupName(findProject.path) === project.name + sanitizeGroupName(findProject.path) === project.name )(allGitlabProjects); // Load default group @@ -176,7 +176,7 @@ const generatePrivateKeyEd25519 = R.partial(generatePrivateKey, ['ed25519']); getSshKeyFingerprint(keyPair.public) ) ); - const userId = R.path([0, 'usid'], userRows); + const userId = R.path([0, 'usid'], userRows) as string; let user; if (!userId) { @@ -209,7 +209,6 @@ const generatePrivateKeyEd25519 = R.partial(generatePrivateKey, ['ed25519']); ); } } else { - // @ts-ignore user = await UserModel.loadUserById(userId); } diff --git a/services/api/src/models/group.ts b/services/api/src/models/group.ts index bad9303a6d..16b443a11c 100644 --- a/services/api/src/models/group.ts +++ b/services/api/src/models/group.ts @@ -1,19 +1,14 @@ -// @ts-ignore import * as R from 'ramda'; -// @ts-ignore import { Pool } from 'mariadb'; -// @ts-ignore import { asyncPipe } from '@lagoon/commons/dist/util/func'; import pickNonNil from '../util/pickNonNil'; import { logger } from '../loggers/logger'; -// @ts-ignore import GroupRepresentation from 'keycloak-admin/lib/defs/groupRepresentation'; import { User } from './user'; import { groupCacheExpiry, get, del, redisClient } from '../clients/redisClient'; import { Helpers as projectHelpers } from '../resources/project/helpers'; import { Helpers as groupHelpers } from '../resources/group/helpers'; import { sqlClientPool } from '../clients/sqlClient'; -import { log } from 'winston'; interface IGroupAttributes { 'lagoon-projects'?: [string]; diff --git a/services/api/src/models/user.ts b/services/api/src/models/user.ts index 7064155b4c..3340b08285 100644 --- a/services/api/src/models/user.ts +++ b/services/api/src/models/user.ts @@ -1,10 +1,8 @@ -// @ts-ignore import * as R from 'ramda'; import pickNonNil from '../util/pickNonNil'; import { logger } from '../loggers/logger'; -// @ts-ignore import UserRepresentation from 'keycloak-admin/lib/defs/userRepresentation'; -import { Group, isRoleSubgroup } from './group'; +import { Group } from './group'; import { sqlClientPool } from '../clients/sqlClient'; import { query } from '../util/db'; import { Sql } from '../resources/user/sql'; diff --git a/services/api/src/resources/deployment/resolvers.ts b/services/api/src/resources/deployment/resolvers.ts index 6cf859f5df..8aa239b80c 100644 --- a/services/api/src/resources/deployment/resolvers.ts +++ b/services/api/src/resources/deployment/resolvers.ts @@ -1,6 +1,4 @@ -// @ts-ignore import * as R from 'ramda'; -// @ts-ignore import { sendToLagoonLogs } from '@lagoon/commons/dist/logs/lagoon-logger'; import { createDeployTask, @@ -8,7 +6,6 @@ import { createPromoteTask, sendToLagoonActions, makeSafe - // @ts-ignore } from '@lagoon/commons/dist/tasks'; import { ResolverFn } from '../'; import { @@ -22,30 +19,20 @@ import { Sql } from './sql'; import { Helpers } from './helpers'; import { Helpers as environmentHelpers } from '../environment/helpers'; import { Helpers as projectHelpers } from '../project/helpers'; -// @ts-ignore import { addTask } from '@lagoon/commons/dist/api'; import { Sql as environmentSql } from '../environment/sql'; -// @ts-ignore import S3 from 'aws-sdk/clients/s3'; -// @ts-ignore import sha1 from 'sha1'; -// @ts-ignore import { generateBuildId } from '@lagoon/commons/dist/util/lagoon'; import { jsonMerge } from '@lagoon/commons/dist/util/func'; import { logger } from '../../loggers/logger'; import { getUserProjectIdsFromRoleProjectIds } from '../../util/auth'; -// @ts-ignore import uuid4 from 'uuid4'; -// @ts-ignore const accessKeyId = process.env.S3_FILES_ACCESS_KEY_ID || 'minio' -// @ts-ignore const secretAccessKey = process.env.S3_FILES_SECRET_ACCESS_KEY || 'minio123' -// @ts-ignore const bucket = process.env.S3_FILES_BUCKET || 'lagoon-files' -// @ts-ignore const region = process.env.S3_FILES_REGION -// @ts-ignore const s3Origin = process.env.S3_FILES_HOST || 'http://docker.for.mac.localhost:9000' const config = { @@ -104,7 +91,6 @@ export const getBuildLog: ResolverFn = async ( if (!data) { return null; } - // @ts-ignore let logMsg = new Buffer(JSON.parse(JSON.stringify(data.Body)).data).toString('ascii'); return logMsg; } catch (e) { diff --git a/services/api/src/resources/env-variables/resolvers.ts b/services/api/src/resources/env-variables/resolvers.ts index 475386514f..8f74d467d9 100644 --- a/services/api/src/resources/env-variables/resolvers.ts +++ b/services/api/src/resources/env-variables/resolvers.ts @@ -1,4 +1,3 @@ -// @ts-ignore import * as R from 'ramda'; import { ResolverFn } from '../'; import { query, knex } from '../../util/db'; @@ -6,7 +5,6 @@ import { Sql } from './sql'; import { Helpers as environmentHelpers } from '../environment/helpers'; import { Helpers as projectHelpers } from '../project/helpers'; import { Sql as projectSql } from '../project/sql'; -import { logger } from '../../loggers/logger'; export const getEnvVarsByProjectId: ResolverFn = async ( @@ -470,4 +468,4 @@ export const getEnvVariablesByProjectEnvironmentName: ResolverFn = async ( } } return []; -}; \ No newline at end of file +}; diff --git a/services/api/src/resources/environment/resolvers.ts b/services/api/src/resources/environment/resolvers.ts index e4ca6a5c54..7878a39d25 100644 --- a/services/api/src/resources/environment/resolvers.ts +++ b/services/api/src/resources/environment/resolvers.ts @@ -1,7 +1,5 @@ import * as R from 'ramda'; -// @ts-ignore import { sendToLagoonLogs } from '@lagoon/commons/dist/logs/lagoon-logger'; -// @ts-ignore import { createRemoveTask } from '@lagoon/commons/dist/tasks'; import { ResolverFn } from '../'; import { logger } from '../../loggers/logger'; @@ -311,7 +309,6 @@ export const addOrUpdateEnvironment: ResolverFn = async ( { sqlClientPool, hasPermission, userActivityLogger } ) => { - // @ts-ignore const pid = input.project.toString(); const openshiftProjectName = input.kubernetesNamespaceName || input.openshiftProjectName; @@ -943,4 +940,4 @@ export const getServiceContainersByServiceId: ResolverFn = async ( Sql.selectContainersByServiceId(sid) ); return await rows; -}; \ No newline at end of file +}; diff --git a/services/api/src/resources/fact/resolvers.ts b/services/api/src/resources/fact/resolvers.ts index 48ee4dc54d..85386493ba 100644 --- a/services/api/src/resources/fact/resolvers.ts +++ b/services/api/src/resources/fact/resolvers.ts @@ -7,9 +7,6 @@ import { ResolverFn } from '../index'; import { knex } from '../../util/db'; import { logger } from '../../loggers/logger'; import crypto from 'crypto'; -import { Service } from 'aws-sdk'; -import * as api from '@lagoon/commons/dist/api'; -import { getEnvironmentsByProjectId } from '../environment/resolvers'; import { getUserProjectIdsFromRoleProjectIds } from '../../util/auth'; export const getFactsByEnvironmentId: ResolverFn = async ( diff --git a/services/api/src/resources/group/resolvers.ts b/services/api/src/resources/group/resolvers.ts index e9afb2cef9..c9792f898f 100644 --- a/services/api/src/resources/group/resolvers.ts +++ b/services/api/src/resources/group/resolvers.ts @@ -1,7 +1,5 @@ -// @ts-ignore import * as R from 'ramda'; import { ResolverFn } from '../'; -// @ts-ignore import validator from 'validator'; import { logger } from '../../loggers/logger'; import { isPatchEmpty } from '../../util/db'; diff --git a/services/api/src/resources/organization/helpers.ts b/services/api/src/resources/organization/helpers.ts index 6647b52ab9..33200337a1 100644 --- a/services/api/src/resources/organization/helpers.ts +++ b/services/api/src/resources/organization/helpers.ts @@ -1,6 +1,4 @@ -// @ts-ignore import * as R from 'ramda'; -// @ts-ignore import { Pool } from 'mariadb'; import { query } from '../../util/db'; import { Sql } from './sql'; @@ -68,4 +66,4 @@ export const Helpers = (sqlClientPool: Pool) => { getNotificationsByTypeForOrganizationId, getEnvironmentsByOrganizationId, } -}; \ No newline at end of file +}; diff --git a/services/api/src/resources/organization/resolvers.ts b/services/api/src/resources/organization/resolvers.ts index b53a85c8fb..93e4dbe3c9 100644 --- a/services/api/src/resources/organization/resolvers.ts +++ b/services/api/src/resources/organization/resolvers.ts @@ -1,4 +1,3 @@ -// @ts-ignore import * as R from 'ramda'; import { ResolverFn } from '../'; import { logger } from '../../loggers/logger'; @@ -11,7 +10,6 @@ import { Helpers as openshiftHelpers } from '../openshift/helpers'; import { Helpers as notificationHelpers } from '../notification/helpers'; import { Helpers as groupHelpers } from '../group/helpers'; import validator from 'validator'; -import { log } from 'winston'; const isValidName = value => { if (validator.matches(value, /[^0-9a-z-]/)) { @@ -1276,4 +1274,4 @@ export const bulkImportProjectsAndGroupsToOrganization: ResolverFn = async ( } return { projects: projectsToMove, groups: groupsToMove, otherOrgProjects: projectsInOtherOrgs, otherOrgGroups: groupsInOtherOrgs }; -} \ No newline at end of file +} diff --git a/services/api/src/resources/project/resolvers.ts b/services/api/src/resources/project/resolvers.ts index 8b916ee8ba..b3a77d8f1e 100644 --- a/services/api/src/resources/project/resolvers.ts +++ b/services/api/src/resources/project/resolvers.ts @@ -1,8 +1,5 @@ -// @ts-ignore import * as R from 'ramda'; -// @ts-ignore import validator from 'validator'; -// @ts-ignore import sshpk from 'sshpk'; import { ResolverFn } from '../'; import { logger } from '../../loggers/logger'; diff --git a/services/api/src/resources/user/resolvers.ts b/services/api/src/resources/user/resolvers.ts index aaf0df8c43..30fa3aa6f6 100644 --- a/services/api/src/resources/user/resolvers.ts +++ b/services/api/src/resources/user/resolvers.ts @@ -1,4 +1,3 @@ -// @ts-ignore import * as R from 'ramda'; import { ResolverFn } from '../'; import { query, isPatchEmpty } from '../../util/db'; diff --git a/services/api/src/util/auth.ts b/services/api/src/util/auth.ts index 4bc3cc5c7a..76289dc70b 100644 --- a/services/api/src/util/auth.ts +++ b/services/api/src/util/auth.ts @@ -1,15 +1,12 @@ -// @ts-ignore import * as R from 'ramda'; import { verify } from 'jsonwebtoken'; import { logger } from '../loggers/logger'; import { getConfigFromEnv } from '../util/config'; import { isNotNil } from './func'; import { keycloakGrantManager } from '../clients/keycloakClient'; -// @ts-ignore const { userActivityLogger } = require('../loggers/userActivityLogger'); import { Group } from '../models/group'; import { User } from '../models/user'; -import { saveRedisKeycloakCache } from '../clients/redisClient'; interface ILegacyToken { iat: string; @@ -326,7 +323,6 @@ export const keycloakHasPermission = (grant, requestCache, modelClients, service authzRequest = { ...authzRequest, claim_token_format: 'urn:ietf:params:oauth:token-type:jwt', - // @ts-ignore claim_token: Buffer.from(JSON.stringify(claims)).toString('base64') }; } @@ -339,7 +335,6 @@ export const keycloakHasPermission = (grant, requestCache, modelClients, service }; try { - // @ts-ignore const newGrant = await keycloakGrantManager.checkPermissions( authzRequest, request diff --git a/services/api/src/util/func.ts b/services/api/src/util/func.ts index 353e0cf0b2..1e21c06174 100644 --- a/services/api/src/util/func.ts +++ b/services/api/src/util/func.ts @@ -1,2 +1 @@ -// @ts-ignore export * from '@lagoon/commons/dist/util/func'; diff --git a/services/webhooks2tasks/src/handlers/problems/processDrutinyResults.ts b/services/webhooks2tasks/src/handlers/problems/processDrutinyResults.ts index 6456811059..beffe3b3ca 100644 --- a/services/webhooks2tasks/src/handlers/problems/processDrutinyResults.ts +++ b/services/webhooks2tasks/src/handlers/problems/processDrutinyResults.ts @@ -1,10 +1,5 @@ // @flow -import { - addProblem, - deleteProblemsFromSource, - getProblemsforProjectEnvironment, -} from'@lagoon/commons/dist/api'; import { sendToLagoonLogs } from '@lagoon/commons/dist/logs/lagoon-logger'; const DRUTINY_VULNERABILITY_SOURCE_BASE = 'Drutiny'; const DRUTINY_SERVICE_NAME = 'cli'; @@ -13,6 +8,9 @@ import { getProjectByName, getEnvironmentByOpenshiftProjectName, getOpenShiftInfoForProject, + addProblem, + deleteProblemsFromSource, + getProblemsforProjectEnvironment, } from '@lagoon/commons/dist/api'; import { generateProblemsWebhookEventName } from "./webhookHelpers"; import * as R from 'ramda';