From 626030c8841fa8fa02a1e2f123a4ba897c60de82 Mon Sep 17 00:00:00 2001 From: Luka Gulin Date: Mon, 4 Apr 2022 22:03:07 +0200 Subject: [PATCH 01/11] Added check for rest of the packages --- packages/dappmanager/src/domains.ts | 6 ++++ .../src/modules/https-portal/index.ts | 15 ++++++++- .../src/modules/installer/runPackages.ts | 31 +++++++++++++++++-- 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/packages/dappmanager/src/domains.ts b/packages/dappmanager/src/domains.ts index 8e47e93b5..19414735d 100644 --- a/packages/dappmanager/src/domains.ts +++ b/packages/dappmanager/src/domains.ts @@ -58,6 +58,12 @@ export function getExternalNetworkAlias(container: ContainerNames): string { return `${shortUniqueDappnodeEns(fullEns)}.external`; } + +export function getExternalNetworkAliasFromPackage(dnpName: string, serviceName: string): string { + const fullEns = getContainerDomain({dnpName, serviceName}); + return `${shortUniqueDappnodeEns(fullEns)}.external`; +} + export function getPublicSubdomain(container: ContainerNames): string { const fullEns = getContainerDomain(container); return stripBadDomainChars(shortUniqueDappnodeEns(fullEns)); diff --git a/packages/dappmanager/src/modules/https-portal/index.ts b/packages/dappmanager/src/modules/https-portal/index.ts index 6daacbad5..d96841426 100644 --- a/packages/dappmanager/src/modules/https-portal/index.ts +++ b/packages/dappmanager/src/modules/https-portal/index.ts @@ -6,7 +6,7 @@ import { } from "../docker"; import { listContainers } from "../docker/list"; import params from "../../params"; -import { getExternalNetworkAlias } from "../../domains"; +import { getExternalNetworkAlias, getExternalNetworkAliasFromPackage } from "../../domains"; import { PackageContainer, HttpsPortalMapping } from "../../types"; import { HttpsPortalApiClient } from "./apiClient"; import { addNetworkAliasCompose, removeNetworkAliasCompose } from "./utils"; @@ -146,6 +146,19 @@ export class HttpsPortal { return mappings; } + async hasMapping(dnpName: string, serviceName: string) : Promise { + + const entries = await this.httpsPortalApiClient.list(); + const mappingAlias = getExternalNetworkAliasFromPackage(dnpName, serviceName); + for (const { fromSubdomain, toHost } of entries) { + const [alias, port] = toHost.split(":"); + if(alias === mappingAlias) { + return true; + } + } + return false; + } + private async getContainerForMapping( mapping: HttpsPortalMapping, containers?: PackageContainer[] diff --git a/packages/dappmanager/src/modules/installer/runPackages.ts b/packages/dappmanager/src/modules/installer/runPackages.ts index 61f1eaf15..13294e50e 100644 --- a/packages/dappmanager/src/modules/installer/runPackages.ts +++ b/packages/dappmanager/src/modules/installer/runPackages.ts @@ -10,6 +10,12 @@ import { dockerComposeUpPackage, dockerCreateNetwork, dockerListNetworks } from import { packageToInstallHasPid } from "../../utils/pid"; import { exposeByDefaultHttpsPorts } from "./exposeByDefaultHttpsPorts"; import { ComposeFileEditor } from "../compose/editor"; +import { getExternalNetworkAliasFromPackage } from "../../domains"; +import { httpsPortal } from "../../calls/httpsPortal"; + + +const externalNetworkName = params.DNP_EXTERNAL_NETWORK_NAME; + /** * Create and run each package container in series @@ -47,7 +53,6 @@ export async function runPackages( log(pkg.dnpName, "Ensuring HTTPS network exists..."); const networks = await dockerListNetworks(); - const externalNetworkName = params.DNP_EXTERNAL_NETWORK_NAME; if (!networks.find(network => network.Name === externalNetworkName)) { await dockerCreateNetwork(externalNetworkName); } @@ -56,12 +61,32 @@ export async function runPackages( const compose = new ComposeFileEditor(pkg.dnpName, true); if(compose.getComposeNetwork(externalNetworkName) === null) { log(pkg.dnpName, "Adding external network to HTTPS compose..."); - const composeService = compose.services()["https.dnp.dappnode.eth"]; - const aliases: string[] = ["https.external"]; + const composeService = compose.firstService(); + const alias = getExternalNetworkAliasFromPackage(pkg.dnpName, composeService.serviceName) + const aliases = [alias]; composeService.addNetwork(externalNetworkName, {aliases}); compose.write(); } + } else { + const compose = new ComposeFileEditor(pkg.dnpName, true); + const services = [compose.firstService()] // TODO: list of all service + + for(const composeService of services) { + if(await httpsPortal.hasMapping(pkg.dnpName, composeService.serviceName)) { + + const networks = await dockerListNetworks(); + if (!networks.find(network => network.Name === externalNetworkName)) { + await dockerCreateNetwork(externalNetworkName); + } + + const alias = getExternalNetworkAliasFromPackage(pkg.dnpName, composeService.serviceName) + const aliases = [alias]; + composeService.addNetwork(externalNetworkName, {aliases}); + compose.write(); + } + } } + await dockerComposeUp(pkg.composePath, { // To clean-up changing multi-service packages, remove orphans From 3ce45663b7618b855b272826e49622511d1f15ac Mon Sep 17 00:00:00 2001 From: pablo Date: Tue, 5 Apr 2022 13:50:07 +0200 Subject: [PATCH 02/11] simplify --- .../src/modules/installer/https.ts | 142 ++++++++++++++++++ .../src/modules/installer/runPackages.ts | 58 ++----- 2 files changed, 152 insertions(+), 48 deletions(-) create mode 100644 packages/dappmanager/src/modules/installer/https.ts diff --git a/packages/dappmanager/src/modules/installer/https.ts b/packages/dappmanager/src/modules/installer/https.ts new file mode 100644 index 000000000..e10a02e81 --- /dev/null +++ b/packages/dappmanager/src/modules/installer/https.ts @@ -0,0 +1,142 @@ +import { listPackageNoThrow } from "../docker/list/listPackages"; +import { httpsPortal } from "../../calls/httpsPortal"; +import { prettyDnpName } from "../../utils/format"; +import params from "../../params"; +import { InstallPackageData } from "../../types"; +import { Log } from "../../utils/logUi"; +import { HttpsPortalMapping } from "../../common"; +import { getExternalNetworkAliasFromPackage } from "../../domains"; +import { ComposeFileEditor } from "../compose/editor"; +import { dockerListNetworks, dockerCreateNetwork } from "../docker"; + +/** + * Recreate HTTPs portal mapping if installing or updating HTTPs package + */ +export async function httpsEnsureNetworkExists( + pkg: InstallPackageData, + externalNetworkName: string +): Promise { + const networks = await dockerListNetworks(); + if (!networks.find(network => network.Name === externalNetworkName)) { + await dockerCreateNetwork(externalNetworkName); + } + + // Check whether DNP_HTTPS compose has external network persisted + const compose = new ComposeFileEditor(pkg.dnpName, true); + if (compose.getComposeNetwork(externalNetworkName) === null) { + const composeService = compose.firstService(); + const alias = getExternalNetworkAliasFromPackage( + pkg.dnpName, + composeService.serviceName + ); + const aliases = [alias]; + composeService.addNetwork(externalNetworkName, { aliases }); + compose.write(); + } +} + +/** + * Persist external network on packages compose files + */ +export async function httpsPersistPackagesExternalNetwork( + pkg: InstallPackageData, + externalNetworkName: string +): Promise { + const compose = new ComposeFileEditor(pkg.dnpName, true); + const services = Object.entries(compose.services()); + + for (const [serviceName, composeServiceEditor] of services) { + if (await httpsPortal.hasMapping(pkg.dnpName, serviceName)) { + const networks = await dockerListNetworks(); + if (!networks.find(network => network.Name === externalNetworkName)) { + await dockerCreateNetwork(externalNetworkName); + } + + const alias = getExternalNetworkAliasFromPackage( + pkg.dnpName, + serviceName + ); + const aliases = [alias]; + composeServiceEditor.addNetwork(externalNetworkName, { aliases }); + compose.write(); + } + } +} + +/** + * Expose default HTTPS ports on installation defined in the manifest - exposable + */ +export async function httpsExposeByDefaultPorts( + pkg: InstallPackageData, + log: Log +): Promise { + if (pkg.metadata.exposable) { + // Check HTTPS package exists + const httpsPackage = await listPackageNoThrow({ + dnpName: params.HTTPS_PORTAL_DNPNAME + }); + if (!httpsPackage) + throw Error( + `HTTPS package not found but required to expose HTTPS ports by default. Install HTTPS package first.` + ); + // Check HTTPS package running + httpsPackage.containers.map(container => { + if (!container.running) + throw Error( + `HTTPS package not running but required to expose HTTPS ports by default.` + ); + }); + const currentMappings = await httpsPortal.getMappings(); + const portMappinRollback: HttpsPortalMapping[] = []; + + for (const exposable of pkg.metadata.exposable) { + if (exposable.exposeByDefault) { + const portalMapping: HttpsPortalMapping = { + fromSubdomain: exposable.fromSubdomain || prettyDnpName(pkg.dnpName), // get dnpName by default + dnpName: pkg.dnpName, + serviceName: + exposable.serviceName || Object.keys(pkg.compose.services)[0], // get first service name by default (docs: https://docs.dappnode.io/es/developers/manifest-reference/#servicename) + port: exposable.port + }; + + if ( + currentMappings.length > 0 && + currentMappings.includes(portalMapping) + ) + continue; + + try { + // Expose default HTTPS ports + log( + pkg.dnpName, + `Exposing ${prettyDnpName(pkg.dnpName)}:${ + exposable.port + } to the external internet` + ); + await httpsPortal.addMapping(portalMapping); + portMappinRollback.push(portalMapping); + + log( + pkg.dnpName, + `Exposed ${prettyDnpName(pkg.dnpName)}:${ + exposable.port + } to the external internet` + ); + } catch (e) { + e.message = `${e.message} Error exposing default HTTPS ports, removing mappings`; + for (const mappingRollback of portMappinRollback) { + await httpsPortal.removeMapping(mappingRollback).catch(e => { + log( + pkg.dnpName, + `Error removing mapping ${JSON.stringify(mappingRollback)}, ${ + e.message + }` + ); + }); + } + throw e; + } + } + } + } +} diff --git a/packages/dappmanager/src/modules/installer/runPackages.ts b/packages/dappmanager/src/modules/installer/runPackages.ts index 13294e50e..a254d62ea 100644 --- a/packages/dappmanager/src/modules/installer/runPackages.ts +++ b/packages/dappmanager/src/modules/installer/runPackages.ts @@ -6,17 +6,16 @@ import { Log } from "../../utils/logUi"; import { copyFileTo } from "../../calls/copyFileTo"; import { InstallPackageData } from "../../types"; import { logs } from "../../logs"; -import { dockerComposeUpPackage, dockerCreateNetwork, dockerListNetworks } from "../docker"; +import { dockerComposeUpPackage } from "../docker"; import { packageToInstallHasPid } from "../../utils/pid"; -import { exposeByDefaultHttpsPorts } from "./exposeByDefaultHttpsPorts"; -import { ComposeFileEditor } from "../compose/editor"; -import { getExternalNetworkAliasFromPackage } from "../../domains"; -import { httpsPortal } from "../../calls/httpsPortal"; - +import { + httpsEnsureNetworkExists, + httpsExposeByDefaultPorts, + httpsPersistPackagesExternalNetwork +} from "./https"; const externalNetworkName = params.DNP_EXTERNAL_NETWORK_NAME; - /** * Create and run each package container in series * The order is extremely important and should be guaranteed by `orderInstallPackages` @@ -47,46 +46,9 @@ export async function runPackages( // - Allow conditionally starting containers latter if were previously running log(pkg.dnpName, "Preparing package..."); - - // Recreate HTTPs portal mapping if installing or updating HTTPs package - if (pkg.dnpName === params.HTTPS_PORTAL_DNPNAME) { - log(pkg.dnpName, "Ensuring HTTPS network exists..."); - - const networks = await dockerListNetworks(); - if (!networks.find(network => network.Name === externalNetworkName)) { - await dockerCreateNetwork(externalNetworkName); - } - - // Check whether DNP_HTTPS compose has external network persisted - const compose = new ComposeFileEditor(pkg.dnpName, true); - if(compose.getComposeNetwork(externalNetworkName) === null) { - log(pkg.dnpName, "Adding external network to HTTPS compose..."); - const composeService = compose.firstService(); - const alias = getExternalNetworkAliasFromPackage(pkg.dnpName, composeService.serviceName) - const aliases = [alias]; - composeService.addNetwork(externalNetworkName, {aliases}); - compose.write(); - } - } else { - const compose = new ComposeFileEditor(pkg.dnpName, true); - const services = [compose.firstService()] // TODO: list of all service - - for(const composeService of services) { - if(await httpsPortal.hasMapping(pkg.dnpName, composeService.serviceName)) { - - const networks = await dockerListNetworks(); - if (!networks.find(network => network.Name === externalNetworkName)) { - await dockerCreateNetwork(externalNetworkName); - } - - const alias = getExternalNetworkAliasFromPackage(pkg.dnpName, composeService.serviceName) - const aliases = [alias]; - composeService.addNetwork(externalNetworkName, {aliases}); - compose.write(); - } - } - } - + if (pkg.dnpName === params.HTTPS_PORTAL_DNPNAME) + await httpsEnsureNetworkExists(pkg, externalNetworkName); + else await httpsPersistPackagesExternalNetwork(pkg, externalNetworkName); await dockerComposeUp(pkg.composePath, { // To clean-up changing multi-service packages, remove orphans @@ -129,6 +91,6 @@ export async function runPackages( log(pkg.dnpName, "Package started"); // Expose default HTTPs ports if required - await exposeByDefaultHttpsPorts(pkg, log); + await httpsExposeByDefaultPorts(pkg, log); } } From 90a3525fb4b8c5b573436cb00dffe06d3567ece7 Mon Sep 17 00:00:00 2001 From: pablo Date: Tue, 5 Apr 2022 13:51:05 +0200 Subject: [PATCH 03/11] added break --- packages/dappmanager/src/modules/installer/https.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/dappmanager/src/modules/installer/https.ts b/packages/dappmanager/src/modules/installer/https.ts index e10a02e81..cb3645f6a 100644 --- a/packages/dappmanager/src/modules/installer/https.ts +++ b/packages/dappmanager/src/modules/installer/https.ts @@ -59,6 +59,7 @@ export async function httpsPersistPackagesExternalNetwork( const aliases = [alias]; composeServiceEditor.addNetwork(externalNetworkName, { aliases }); compose.write(); + break; } } } From 6d1254964a5d2ce7ac79d9961471abd8b278dfa1 Mon Sep 17 00:00:00 2001 From: Luka Gulin Date: Tue, 5 Apr 2022 14:49:56 +0200 Subject: [PATCH 04/11] bugfix --- packages/dappmanager/src/modules/installer/https.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dappmanager/src/modules/installer/https.ts b/packages/dappmanager/src/modules/installer/https.ts index cb3645f6a..5584e2006 100644 --- a/packages/dappmanager/src/modules/installer/https.ts +++ b/packages/dappmanager/src/modules/installer/https.ts @@ -42,7 +42,7 @@ export async function httpsPersistPackagesExternalNetwork( pkg: InstallPackageData, externalNetworkName: string ): Promise { - const compose = new ComposeFileEditor(pkg.dnpName, true); + const compose = new ComposeFileEditor(pkg.dnpName, pkg.isCore); const services = Object.entries(compose.services()); for (const [serviceName, composeServiceEditor] of services) { From df50b6ccf33f5814a610b1a5010d7fa28a2442fa Mon Sep 17 00:00:00 2001 From: Luka Gulin Date: Tue, 5 Apr 2022 15:19:31 +0200 Subject: [PATCH 05/11] Check if https exists --- .../src/modules/installer/https.ts | 51 ++++++++++++++----- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/packages/dappmanager/src/modules/installer/https.ts b/packages/dappmanager/src/modules/installer/https.ts index 5584e2006..b81e0cf7d 100644 --- a/packages/dappmanager/src/modules/installer/https.ts +++ b/packages/dappmanager/src/modules/installer/https.ts @@ -42,6 +42,10 @@ export async function httpsPersistPackagesExternalNetwork( pkg: InstallPackageData, externalNetworkName: string ): Promise { + + if(!hasRunningHTTPS()) { // if there is no https, checks aren't needed + return; + } const compose = new ComposeFileEditor(pkg.dnpName, pkg.isCore); const services = Object.entries(compose.services()); @@ -73,20 +77,9 @@ export async function httpsExposeByDefaultPorts( ): Promise { if (pkg.metadata.exposable) { // Check HTTPS package exists - const httpsPackage = await listPackageNoThrow({ - dnpName: params.HTTPS_PORTAL_DNPNAME - }); - if (!httpsPackage) - throw Error( - `HTTPS package not found but required to expose HTTPS ports by default. Install HTTPS package first.` - ); - // Check HTTPS package running - httpsPackage.containers.map(container => { - if (!container.running) - throw Error( - `HTTPS package not running but required to expose HTTPS ports by default.` - ); - }); + + hasRunningHTTPS(true); // require that https package exists and it is running + const currentMappings = await httpsPortal.getMappings(); const portMappinRollback: HttpsPortalMapping[] = []; @@ -141,3 +134,33 @@ export async function httpsExposeByDefaultPorts( } } } + +async function hasRunningHTTPS(require: boolean = false) { + + const httpsPackage = await listPackageNoThrow({ + dnpName: params.HTTPS_PORTAL_DNPNAME + }); + if (!httpsPackage) { + if(require) { + return false; + } + + throw Error( + `HTTPS package not found but required to expose HTTPS ports by default. Install HTTPS package first.` + ); + } + + // Check HTTPS package running + httpsPackage.containers.forEach(container => { + if (!container.running){ + if(require) { + return false; + } + throw Error( + `HTTPS package not running but required to expose HTTPS ports by default.` + ); + } + }); + + return true; +} \ No newline at end of file From ff880ca51862a7f3f5523fde433c6822f7689a2e Mon Sep 17 00:00:00 2001 From: Luka Gulin Date: Tue, 5 Apr 2022 17:48:04 +0200 Subject: [PATCH 06/11] Await promises --- packages/dappmanager/src/modules/installer/https.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/dappmanager/src/modules/installer/https.ts b/packages/dappmanager/src/modules/installer/https.ts index b81e0cf7d..2c69f3aa4 100644 --- a/packages/dappmanager/src/modules/installer/https.ts +++ b/packages/dappmanager/src/modules/installer/https.ts @@ -43,7 +43,7 @@ export async function httpsPersistPackagesExternalNetwork( externalNetworkName: string ): Promise { - if(!hasRunningHTTPS()) { // if there is no https, checks aren't needed + if(!(await hasRunningHTTPS())) { // if there is no https, checks aren't needed return; } const compose = new ComposeFileEditor(pkg.dnpName, pkg.isCore); @@ -78,7 +78,7 @@ export async function httpsExposeByDefaultPorts( if (pkg.metadata.exposable) { // Check HTTPS package exists - hasRunningHTTPS(true); // require that https package exists and it is running + await hasRunningHTTPS(true); // require that https package exists and it is running const currentMappings = await httpsPortal.getMappings(); const portMappinRollback: HttpsPortalMapping[] = []; @@ -141,7 +141,7 @@ async function hasRunningHTTPS(require: boolean = false) { dnpName: params.HTTPS_PORTAL_DNPNAME }); if (!httpsPackage) { - if(require) { + if(!require) { return false; } @@ -153,7 +153,7 @@ async function hasRunningHTTPS(require: boolean = false) { // Check HTTPS package running httpsPackage.containers.forEach(container => { if (!container.running){ - if(require) { + if(!require) { return false; } throw Error( From 48037a6010fedb3e1adf9a7a4cab3c6098c8fd47 Mon Sep 17 00:00:00 2001 From: Luka Gulin Date: Wed, 6 Apr 2022 11:38:39 +0200 Subject: [PATCH 07/11] Change require to required --- packages/dappmanager/src/modules/installer/https.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/dappmanager/src/modules/installer/https.ts b/packages/dappmanager/src/modules/installer/https.ts index 2c69f3aa4..911303fb1 100644 --- a/packages/dappmanager/src/modules/installer/https.ts +++ b/packages/dappmanager/src/modules/installer/https.ts @@ -135,13 +135,13 @@ export async function httpsExposeByDefaultPorts( } } -async function hasRunningHTTPS(require: boolean = false) { +async function hasRunningHTTPS(required: boolean = false) { const httpsPackage = await listPackageNoThrow({ dnpName: params.HTTPS_PORTAL_DNPNAME }); if (!httpsPackage) { - if(!require) { + if(!required) { return false; } @@ -153,7 +153,7 @@ async function hasRunningHTTPS(require: boolean = false) { // Check HTTPS package running httpsPackage.containers.forEach(container => { if (!container.running){ - if(!require) { + if(!required) { return false; } throw Error( From 43516d3750a6c701e0180b4f5d431b4fe7202d6e Mon Sep 17 00:00:00 2001 From: Luka Gulin Date: Sun, 10 Apr 2022 22:20:09 +0200 Subject: [PATCH 08/11] Review fixes --- .../src/modules/https-portal/index.ts | 6 ++-- .../src/modules/installer/https.ts | 32 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/packages/dappmanager/src/modules/https-portal/index.ts b/packages/dappmanager/src/modules/https-portal/index.ts index d96841426..32029a1ce 100644 --- a/packages/dappmanager/src/modules/https-portal/index.ts +++ b/packages/dappmanager/src/modules/https-portal/index.ts @@ -74,7 +74,7 @@ export class HttpsPortal { const httpsComposePath = ComposeEditor.getComposePath(params.HTTPS_PORTAL_DNPNAME, true) const editor = new ComposeEditor(ComposeEditor.readFrom(httpsComposePath)) - if(editor.getComposeNetwork(externalNetworkName) === null) { + if (editor.getComposeNetwork(externalNetworkName) === null) { const httpsExternalAlias = getExternalNetworkAlias(httpsPortalContainer) addNetworkAliasCompose(httpsPortalContainer, externalNetworkName, [httpsExternalAlias]) } @@ -146,13 +146,13 @@ export class HttpsPortal { return mappings; } - async hasMapping(dnpName: string, serviceName: string) : Promise { + async hasMapping(dnpName: string, serviceName: string): Promise { const entries = await this.httpsPortalApiClient.list(); const mappingAlias = getExternalNetworkAliasFromPackage(dnpName, serviceName); for (const { fromSubdomain, toHost } of entries) { const [alias, port] = toHost.split(":"); - if(alias === mappingAlias) { + if (alias === mappingAlias) { return true; } } diff --git a/packages/dappmanager/src/modules/installer/https.ts b/packages/dappmanager/src/modules/installer/https.ts index 911303fb1..be2fac4f3 100644 --- a/packages/dappmanager/src/modules/installer/https.ts +++ b/packages/dappmanager/src/modules/installer/https.ts @@ -16,13 +16,19 @@ export async function httpsEnsureNetworkExists( pkg: InstallPackageData, externalNetworkName: string ): Promise { + + if (pkg.dnpName !== params.HTTPS_PORTAL_DNPNAME) { + throw Error( + `package must be DNP_HTTPS.` + ); + } const networks = await dockerListNetworks(); if (!networks.find(network => network.Name === externalNetworkName)) { await dockerCreateNetwork(externalNetworkName); } // Check whether DNP_HTTPS compose has external network persisted - const compose = new ComposeFileEditor(pkg.dnpName, true); + const compose = new ComposeFileEditor(pkg.dnpName, pkg.isCore); if (compose.getComposeNetwork(externalNetworkName) === null) { const composeService = compose.firstService(); const alias = getExternalNetworkAliasFromPackage( @@ -43,7 +49,7 @@ export async function httpsPersistPackagesExternalNetwork( externalNetworkName: string ): Promise { - if(!(await hasRunningHTTPS())) { // if there is no https, checks aren't needed + if (!(await hasRunningHTTPS())) { // if there is no https, checks aren't needed return; } const compose = new ComposeFileEditor(pkg.dnpName, pkg.isCore); @@ -103,18 +109,14 @@ export async function httpsExposeByDefaultPorts( // Expose default HTTPS ports log( pkg.dnpName, - `Exposing ${prettyDnpName(pkg.dnpName)}:${ - exposable.port - } to the external internet` + `Exposing ${prettyDnpName(pkg.dnpName)}:${exposable.port} to the external internet` ); await httpsPortal.addMapping(portalMapping); portMappinRollback.push(portalMapping); log( pkg.dnpName, - `Exposed ${prettyDnpName(pkg.dnpName)}:${ - exposable.port - } to the external internet` + `Exposed ${prettyDnpName(pkg.dnpName)}:${exposable.port} to the external internet` ); } catch (e) { e.message = `${e.message} Error exposing default HTTPS ports, removing mappings`; @@ -122,9 +124,7 @@ export async function httpsExposeByDefaultPorts( await httpsPortal.removeMapping(mappingRollback).catch(e => { log( pkg.dnpName, - `Error removing mapping ${JSON.stringify(mappingRollback)}, ${ - e.message - }` + `Error removing mapping ${JSON.stringify(mappingRollback)}, ${e.message}` ); }); } @@ -141,7 +141,7 @@ async function hasRunningHTTPS(required: boolean = false) { dnpName: params.HTTPS_PORTAL_DNPNAME }); if (!httpsPackage) { - if(!required) { + if (!required) { return false; } @@ -149,11 +149,11 @@ async function hasRunningHTTPS(required: boolean = false) { `HTTPS package not found but required to expose HTTPS ports by default. Install HTTPS package first.` ); } - + // Check HTTPS package running httpsPackage.containers.forEach(container => { - if (!container.running){ - if(!required) { + if (!container.running) { + if (!required) { return false; } throw Error( @@ -161,6 +161,6 @@ async function hasRunningHTTPS(required: boolean = false) { ); } }); - + return true; } \ No newline at end of file From 35cc2f122e35ad37dba3b6a61011f77eaf02a3d1 Mon Sep 17 00:00:00 2001 From: 3alpha Date: Tue, 31 May 2022 16:31:34 +0200 Subject: [PATCH 09/11] Simplify adding network to HTTPS --- .../src/modules/installer/https.ts | 35 +++++++++---------- .../src/modules/installer/runPackages.ts | 16 +++++---- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/packages/dappmanager/src/modules/installer/https.ts b/packages/dappmanager/src/modules/installer/https.ts index be2fac4f3..230a316c9 100644 --- a/packages/dappmanager/src/modules/installer/https.ts +++ b/packages/dappmanager/src/modules/installer/https.ts @@ -7,38 +7,37 @@ import { Log } from "../../utils/logUi"; import { HttpsPortalMapping } from "../../common"; import { getExternalNetworkAliasFromPackage } from "../../domains"; import { ComposeFileEditor } from "../compose/editor"; -import { dockerListNetworks, dockerCreateNetwork } from "../docker"; +import { dockerListNetworks, dockerCreateNetwork, dockerNetworkConnect } from "../docker"; /** * Recreate HTTPs portal mapping if installing or updating HTTPs package */ export async function httpsEnsureNetworkExists( - pkg: InstallPackageData, externalNetworkName: string ): Promise { - if (pkg.dnpName !== params.HTTPS_PORTAL_DNPNAME) { - throw Error( - `package must be DNP_HTTPS.` - ); + const httpsPackage = await listPackageNoThrow({ + dnpName: params.HTTPS_PORTAL_DNPNAME + }); + + if (!(await hasRunningHTTPS())) { // if there is no https, checks aren't needed + return; } + const networks = await dockerListNetworks(); + if (!networks.find(network => network.Name === externalNetworkName)) { await dockerCreateNetwork(externalNetworkName); } - // Check whether DNP_HTTPS compose has external network persisted - const compose = new ComposeFileEditor(pkg.dnpName, pkg.isCore); - if (compose.getComposeNetwork(externalNetworkName) === null) { - const composeService = compose.firstService(); - const alias = getExternalNetworkAliasFromPackage( - pkg.dnpName, - composeService.serviceName - ); - const aliases = [alias]; - composeService.addNetwork(externalNetworkName, { aliases }); - compose.write(); - } + const containers = httpsPackage?.containers ?? [] + + await Promise.all( + containers.map(async (container) => { + if(!container.networks.find(n => n.name === externalNetworkName)) { + await dockerNetworkConnect(externalNetworkName, container.containerName) + } + })) } /** diff --git a/packages/dappmanager/src/modules/installer/runPackages.ts b/packages/dappmanager/src/modules/installer/runPackages.ts index a254d62ea..70629bf49 100644 --- a/packages/dappmanager/src/modules/installer/runPackages.ts +++ b/packages/dappmanager/src/modules/installer/runPackages.ts @@ -45,11 +45,6 @@ export async function runPackages( // - Allow copying files without duplicating logic // - Allow conditionally starting containers latter if were previously running log(pkg.dnpName, "Preparing package..."); - - if (pkg.dnpName === params.HTTPS_PORTAL_DNPNAME) - await httpsEnsureNetworkExists(pkg, externalNetworkName); - else await httpsPersistPackagesExternalNetwork(pkg, externalNetworkName); - await dockerComposeUp(pkg.composePath, { // To clean-up changing multi-service packages, remove orphans // but NOT for core packages, which always have orphans @@ -91,6 +86,15 @@ export async function runPackages( log(pkg.dnpName, "Package started"); // Expose default HTTPs ports if required - await httpsExposeByDefaultPorts(pkg, log); + + if (pkg.dnpName === params.HTTPS_PORTAL_DNPNAME) { + await httpsEnsureNetworkExists(externalNetworkName); + } + + else { + await httpsExposeByDefaultPorts(pkg, log); + await httpsPersistPackagesExternalNetwork(pkg, externalNetworkName); + } + } } From 432a4cd156174e3fcdc44a4fe3c0c623549b9f7b Mon Sep 17 00:00:00 2001 From: 3alpha Date: Tue, 31 May 2022 17:33:26 +0200 Subject: [PATCH 10/11] Simplify #2 --- .../src/modules/installer/https.ts | 56 +++++-------------- .../src/modules/installer/runPackages.ts | 15 ++--- 2 files changed, 18 insertions(+), 53 deletions(-) diff --git a/packages/dappmanager/src/modules/installer/https.ts b/packages/dappmanager/src/modules/installer/https.ts index 230a316c9..44fabef79 100644 --- a/packages/dappmanager/src/modules/installer/https.ts +++ b/packages/dappmanager/src/modules/installer/https.ts @@ -9,66 +9,38 @@ import { getExternalNetworkAliasFromPackage } from "../../domains"; import { ComposeFileEditor } from "../compose/editor"; import { dockerListNetworks, dockerCreateNetwork, dockerNetworkConnect } from "../docker"; + /** - * Recreate HTTPs portal mapping if installing or updating HTTPs package + * Persist external network on packages compose files */ -export async function httpsEnsureNetworkExists( +export async function connectToPublicNetwork( + pkg: InstallPackageData, externalNetworkName: string ): Promise { - const httpsPackage = await listPackageNoThrow({ - dnpName: params.HTTPS_PORTAL_DNPNAME - }); - if (!(await hasRunningHTTPS())) { // if there is no https, checks aren't needed return; } const networks = await dockerListNetworks(); - if (!networks.find(network => network.Name === externalNetworkName)) { await dockerCreateNetwork(externalNetworkName); } - const containers = httpsPackage?.containers ?? [] - - await Promise.all( - containers.map(async (container) => { - if(!container.networks.find(n => n.name === externalNetworkName)) { - await dockerNetworkConnect(externalNetworkName, container.containerName) - } - })) -} - -/** - * Persist external network on packages compose files - */ -export async function httpsPersistPackagesExternalNetwork( - pkg: InstallPackageData, - externalNetworkName: string -): Promise { - - if (!(await hasRunningHTTPS())) { // if there is no https, checks aren't needed - return; - } - const compose = new ComposeFileEditor(pkg.dnpName, pkg.isCore); - const services = Object.entries(compose.services()); - - for (const [serviceName, composeServiceEditor] of services) { - if (await httpsPortal.hasMapping(pkg.dnpName, serviceName)) { - const networks = await dockerListNetworks(); - if (!networks.find(network => network.Name === externalNetworkName)) { - await dockerCreateNetwork(externalNetworkName); - } + const containers = (await listPackageNoThrow({ + dnpName: pkg.dnpName + }))?.containers || []; + for(const container of containers) { + if (pkg.dnpName === params.HTTPS_PORTAL_DNPNAME || await httpsPortal.hasMapping(pkg.dnpName, container.serviceName)) { const alias = getExternalNetworkAliasFromPackage( pkg.dnpName, - serviceName + container.serviceName ); - const aliases = [alias]; - composeServiceEditor.addNetwork(externalNetworkName, { aliases }); - compose.write(); - break; + + if(!container.networks.find(n => n.name === externalNetworkName)) { + await dockerNetworkConnect(externalNetworkName, container.containerName, {Aliases: [alias]}) + } } } } diff --git a/packages/dappmanager/src/modules/installer/runPackages.ts b/packages/dappmanager/src/modules/installer/runPackages.ts index 70629bf49..a81f60ec8 100644 --- a/packages/dappmanager/src/modules/installer/runPackages.ts +++ b/packages/dappmanager/src/modules/installer/runPackages.ts @@ -9,9 +9,8 @@ import { logs } from "../../logs"; import { dockerComposeUpPackage } from "../docker"; import { packageToInstallHasPid } from "../../utils/pid"; import { - httpsEnsureNetworkExists, + connectToPublicNetwork, httpsExposeByDefaultPorts, - httpsPersistPackagesExternalNetwork } from "./https"; const externalNetworkName = params.DNP_EXTERNAL_NETWORK_NAME; @@ -87,14 +86,8 @@ export async function runPackages( // Expose default HTTPs ports if required - if (pkg.dnpName === params.HTTPS_PORTAL_DNPNAME) { - await httpsEnsureNetworkExists(externalNetworkName); - } - - else { - await httpsExposeByDefaultPorts(pkg, log); - await httpsPersistPackagesExternalNetwork(pkg, externalNetworkName); - } - + await connectToPublicNetwork(pkg, externalNetworkName); + await httpsExposeByDefaultPorts(pkg, log); + } } From 67e4da748d1e199ad5c8606f1cd3d0fc3b822d11 Mon Sep 17 00:00:00 2001 From: Pablo Date: Wed, 1 Jun 2022 11:23:01 +0200 Subject: [PATCH 11/11] clean code --- packages/dappmanager/src/domains.ts | 6 - .../src/modules/https-portal/index.ts | 31 +++-- .../src/modules/installer/https.ts | 112 ++++++++++-------- .../src/modules/installer/runPackages.ts | 10 +- 4 files changed, 83 insertions(+), 76 deletions(-) diff --git a/packages/dappmanager/src/domains.ts b/packages/dappmanager/src/domains.ts index 19414735d..8e47e93b5 100644 --- a/packages/dappmanager/src/domains.ts +++ b/packages/dappmanager/src/domains.ts @@ -58,12 +58,6 @@ export function getExternalNetworkAlias(container: ContainerNames): string { return `${shortUniqueDappnodeEns(fullEns)}.external`; } - -export function getExternalNetworkAliasFromPackage(dnpName: string, serviceName: string): string { - const fullEns = getContainerDomain({dnpName, serviceName}); - return `${shortUniqueDappnodeEns(fullEns)}.external`; -} - export function getPublicSubdomain(container: ContainerNames): string { const fullEns = getContainerDomain(container); return stripBadDomainChars(shortUniqueDappnodeEns(fullEns)); diff --git a/packages/dappmanager/src/modules/https-portal/index.ts b/packages/dappmanager/src/modules/https-portal/index.ts index 32029a1ce..69404edb0 100644 --- a/packages/dappmanager/src/modules/https-portal/index.ts +++ b/packages/dappmanager/src/modules/https-portal/index.ts @@ -6,7 +6,7 @@ import { } from "../docker"; import { listContainers } from "../docker/list"; import params from "../../params"; -import { getExternalNetworkAlias, getExternalNetworkAliasFromPackage } from "../../domains"; +import { getExternalNetworkAlias } from "../../domains"; import { PackageContainer, HttpsPortalMapping } from "../../types"; import { HttpsPortalApiClient } from "./apiClient"; import { addNetworkAliasCompose, removeNetworkAliasCompose } from "./utils"; @@ -69,14 +69,18 @@ export class HttpsPortal { // Edit compose to persist the setting addNetworkAliasCompose(container, externalNetworkName, aliases); - // Check whether DNP_HTTPS compose has external network persisted - const httpsComposePath = ComposeEditor.getComposePath(params.HTTPS_PORTAL_DNPNAME, true) - const editor = new ComposeEditor(ComposeEditor.readFrom(httpsComposePath)) + const httpsComposePath = ComposeEditor.getComposePath( + params.HTTPS_PORTAL_DNPNAME, + true + ); + const editor = new ComposeEditor(ComposeEditor.readFrom(httpsComposePath)); if (editor.getComposeNetwork(externalNetworkName) === null) { - const httpsExternalAlias = getExternalNetworkAlias(httpsPortalContainer) - addNetworkAliasCompose(httpsPortalContainer, externalNetworkName, [httpsExternalAlias]) + const httpsExternalAlias = getExternalNetworkAlias(httpsPortalContainer); + addNetworkAliasCompose(httpsPortalContainer, externalNetworkName, [ + httpsExternalAlias + ]); } } @@ -146,15 +150,16 @@ export class HttpsPortal { return mappings; } + /** + * Returns true if the container has assigned a mapping to the https-portal + */ async hasMapping(dnpName: string, serviceName: string): Promise { - const entries = await this.httpsPortalApiClient.list(); - const mappingAlias = getExternalNetworkAliasFromPackage(dnpName, serviceName); - for (const { fromSubdomain, toHost } of entries) { - const [alias, port] = toHost.split(":"); - if (alias === mappingAlias) { - return true; - } + const mappingAlias = getExternalNetworkAlias({ serviceName, dnpName }); + for (const { toHost } of entries) { + // toHost format: someDomain:80 + const alias = toHost.split(":")[0]; + if (alias === mappingAlias) return true; } return false; } diff --git a/packages/dappmanager/src/modules/installer/https.ts b/packages/dappmanager/src/modules/installer/https.ts index 44fabef79..814ad7635 100644 --- a/packages/dappmanager/src/modules/installer/https.ts +++ b/packages/dappmanager/src/modules/installer/https.ts @@ -5,41 +5,55 @@ import params from "../../params"; import { InstallPackageData } from "../../types"; import { Log } from "../../utils/logUi"; import { HttpsPortalMapping } from "../../common"; -import { getExternalNetworkAliasFromPackage } from "../../domains"; -import { ComposeFileEditor } from "../compose/editor"; -import { dockerListNetworks, dockerCreateNetwork, dockerNetworkConnect } from "../docker"; - +import { getExternalNetworkAlias } from "../../domains"; +import { + dockerListNetworks, + dockerCreateNetwork, + dockerNetworkConnect +} from "../docker"; /** - * Persist external network on packages compose files + * Connect to dnpublic_network with an alias if: + * - is HTTPS package + * - any package with https portal mappings */ export async function connectToPublicNetwork( pkg: InstallPackageData, externalNetworkName: string ): Promise { + // if there is no https, checks aren't needed + if (!(await isRunningHttps())) return; - if (!(await hasRunningHTTPS())) { // if there is no https, checks aren't needed - return; - } - + // create network if necessary const networks = await dockerListNetworks(); - if (!networks.find(network => network.Name === externalNetworkName)) { + if (!networks.find(network => network.Name === externalNetworkName)) await dockerCreateNetwork(externalNetworkName); - } - - const containers = (await listPackageNoThrow({ - dnpName: pkg.dnpName - }))?.containers || []; - for(const container of containers) { - if (pkg.dnpName === params.HTTPS_PORTAL_DNPNAME || await httpsPortal.hasMapping(pkg.dnpName, container.serviceName)) { - const alias = getExternalNetworkAliasFromPackage( - pkg.dnpName, - container.serviceName - ); - - if(!container.networks.find(n => n.name === externalNetworkName)) { - await dockerNetworkConnect(externalNetworkName, container.containerName, {Aliases: [alias]}) + const containers = + ( + await listPackageNoThrow({ + dnpName: pkg.dnpName + }) + )?.containers || []; + + if (containers.length === 0) return; + + for (const container of containers) { + if ( + pkg.dnpName === params.HTTPS_PORTAL_DNPNAME || + (await httpsPortal.hasMapping(pkg.dnpName, container.serviceName)) + ) { + const alias = getExternalNetworkAlias({ + serviceName: container.serviceName, + dnpName: pkg.dnpName + }); + + if (!container.networks.find(n => n.name === externalNetworkName)) { + await dockerNetworkConnect( + externalNetworkName, + container.containerName, + { Aliases: [alias] } + ); } } } @@ -48,14 +62,16 @@ export async function connectToPublicNetwork( /** * Expose default HTTPS ports on installation defined in the manifest - exposable */ -export async function httpsExposeByDefaultPorts( +export async function exposeByDefaultHttpsPorts( pkg: InstallPackageData, log: Log ): Promise { if (pkg.metadata.exposable) { - // Check HTTPS package exists - - await hasRunningHTTPS(true); // require that https package exists and it is running + // Requires that https package exists and it is running + if (!(await isRunningHttps())) + throw Error( + `HTTPS package not running but required to expose HTTPS ports by default.` + ); const currentMappings = await httpsPortal.getMappings(); const portMappinRollback: HttpsPortalMapping[] = []; @@ -80,14 +96,18 @@ export async function httpsExposeByDefaultPorts( // Expose default HTTPS ports log( pkg.dnpName, - `Exposing ${prettyDnpName(pkg.dnpName)}:${exposable.port} to the external internet` + `Exposing ${prettyDnpName(pkg.dnpName)}:${ + exposable.port + } to the external internet` ); await httpsPortal.addMapping(portalMapping); portMappinRollback.push(portalMapping); log( pkg.dnpName, - `Exposed ${prettyDnpName(pkg.dnpName)}:${exposable.port} to the external internet` + `Exposed ${prettyDnpName(pkg.dnpName)}:${ + exposable.port + } to the external internet` ); } catch (e) { e.message = `${e.message} Error exposing default HTTPS ports, removing mappings`; @@ -95,7 +115,9 @@ export async function httpsExposeByDefaultPorts( await httpsPortal.removeMapping(mappingRollback).catch(e => { log( pkg.dnpName, - `Error removing mapping ${JSON.stringify(mappingRollback)}, ${e.message}` + `Error removing mapping ${JSON.stringify(mappingRollback)}, ${ + e.message + }` ); }); } @@ -106,32 +128,22 @@ export async function httpsExposeByDefaultPorts( } } -async function hasRunningHTTPS(required: boolean = false) { +// Utils +/** + * Returns true if HTTPS package installed and running, otherwise return false + */ +async function isRunningHttps() { const httpsPackage = await listPackageNoThrow({ dnpName: params.HTTPS_PORTAL_DNPNAME }); - if (!httpsPackage) { - if (!required) { - return false; - } - throw Error( - `HTTPS package not found but required to expose HTTPS ports by default. Install HTTPS package first.` - ); - } + if (!httpsPackage) return false; - // Check HTTPS package running + // Check every HTTPS container is running httpsPackage.containers.forEach(container => { - if (!container.running) { - if (!required) { - return false; - } - throw Error( - `HTTPS package not running but required to expose HTTPS ports by default.` - ); - } + if (!container.running) return false; }); return true; -} \ No newline at end of file +} diff --git a/packages/dappmanager/src/modules/installer/runPackages.ts b/packages/dappmanager/src/modules/installer/runPackages.ts index a81f60ec8..fe17985c2 100644 --- a/packages/dappmanager/src/modules/installer/runPackages.ts +++ b/packages/dappmanager/src/modules/installer/runPackages.ts @@ -8,10 +8,7 @@ import { InstallPackageData } from "../../types"; import { logs } from "../../logs"; import { dockerComposeUpPackage } from "../docker"; import { packageToInstallHasPid } from "../../utils/pid"; -import { - connectToPublicNetwork, - httpsExposeByDefaultPorts, -} from "./https"; +import { connectToPublicNetwork, exposeByDefaultHttpsPorts } from "./https"; const externalNetworkName = params.DNP_EXTERNAL_NETWORK_NAME; @@ -85,9 +82,8 @@ export async function runPackages( log(pkg.dnpName, "Package started"); // Expose default HTTPs ports if required - + await connectToPublicNetwork(pkg, externalNetworkName); - await httpsExposeByDefaultPorts(pkg, log); - + await exposeByDefaultHttpsPorts(pkg, log); } }