From c83556b5f3acb4a3f1588cae99769a49cfe96eaf Mon Sep 17 00:00:00 2001 From: Johannes Rudolph Date: Mon, 2 May 2022 18:36:26 +0200 Subject: [PATCH] feat: implement tenant list command using foundation config --- src/api/CliApiFacadeFactory.ts | 40 +++++--- src/commands/foundation/new.command.ts | 21 ++-- src/commands/tenant/list.command.ts | 36 +++++-- src/mesh/MeshFoundationAdapterFactory.ts | 121 ++++++++++++++++++++++ src/mesh/query-statistics.ts | 3 + src/model/CliToolEnv.ts | 16 +-- src/model/FoundationRepository.ts | 124 +++++++++++++++++++++++ src/process/DefaultEnvShellRunner.ts | 16 +++ src/process/shell-runner.ts | 12 ++- 9 files changed, 344 insertions(+), 45 deletions(-) create mode 100644 src/mesh/MeshFoundationAdapterFactory.ts create mode 100644 src/model/FoundationRepository.ts create mode 100644 src/process/DefaultEnvShellRunner.ts diff --git a/src/api/CliApiFacadeFactory.ts b/src/api/CliApiFacadeFactory.ts index 329a1aeb..7d282c75 100644 --- a/src/api/CliApiFacadeFactory.ts +++ b/src/api/CliApiFacadeFactory.ts @@ -3,11 +3,12 @@ import { Logger } from "../cli/Logger.ts"; import { CmdGlobalOptions } from "../commands/cmd-options.ts"; import { isatty, TTY } from "../commands/tty.ts"; import { MeshError } from "../errors.ts"; +import { AwsCliEnv, AzCliEnv, GcloudCliEnv } from "../model/CliToolEnv.ts"; import { isWindows } from "../os.ts"; +import { DefaultEnvShellRunner } from "../process/DefaultEnvShellRunner.ts"; import { LoaderShellRunner } from "../process/loader-shell-runner.ts"; import { VerboseShellRunner } from "../process/verbose-shell-runner.ts"; import { AwsCliFacade } from "./aws/aws-cli-facade.ts"; -import { AwsShellRunner } from "./aws/aws-shell-runner.ts"; import { AutoInstallAzureCliModuleDecorator } from "./az/auto-install-azure-cli-module-decorator.ts"; import { AzureCliFacade } from "./az/azure-cli-facade.ts"; import { BasicAzureCliFacade } from "./az/basic-azure-cli-facade.ts"; @@ -21,22 +22,22 @@ export class CliApiFacadeFactory { constructor( private readonly logger: Logger, - private readonly options: CmdGlobalOptions + private readonly options: CmdGlobalOptions, ) {} - async buildAws() { - const shellRunner = this.buildShellRunner(this.options); - const awsShellRunner = new AwsShellRunner(shellRunner, "default"); + async buildAws(env?: AwsCliEnv) { + const shellRunner = this.buildShellRunner(this.options, env); + // const awsShellRunner = new AwsShellRunner(shellRunner, "default"); - const facade = new AwsCliFacade(awsShellRunner); + const facade = new AwsCliFacade(shellRunner); await this.verifyInstallationStatus(facade); return facade; } - async buildGcloud() { - const shellRunner = this.buildShellRunner(this.options); + async buildGcloud(env?: GcloudCliEnv) { + const shellRunner = this.buildShellRunner(this.options, env); const facade = new GcpCliFacade(shellRunner); @@ -45,8 +46,8 @@ export class CliApiFacadeFactory { return facade; } - async buildAz() { - const shellRunner = this.buildShellRunner(this.options); + async buildAz(env?: AzCliEnv) { + const shellRunner = this.buildShellRunner(this.options, env); let azure: AzureCliFacade = new BasicAzureCliFacade(shellRunner); @@ -64,12 +65,13 @@ export class CliApiFacadeFactory { private async verifyInstallationStatus(facade: CliFacade) { // maintain a cache of installation status so we don't run any checks again unnecessarily - const key = Object.getPrototypeOf(facade).name; + const key = facade.constructor.name; const cachedStatus = this.installationStatusCache.get(key); this.logger.debug( () => - `requested ${key} has a cached installation status of ${cachedStatus?.status}` + `requested ${key} has a cached installation status of ${cachedStatus + ?.status}`, ); const status = cachedStatus || (await facade.verifyCliInstalled()); @@ -84,18 +86,21 @@ export class CliApiFacadeFactory { switch (status.status) { case PlatformCommandInstallationStatus.NotInstalled: throw new MeshError( - `"${cmd}" cli is not installed. Please review https://github.com/meshcloud/collie-cli/#prerequisites for installation instructions".` + `"${cmd}" cli is not installed. Please review https://github.com/meshcloud/collie-cli/#prerequisites for installation instructions".`, ); case PlatformCommandInstallationStatus.UnsupportedVersion: throw new MeshError( - `"${cmd}" cli is not installed in a supported version. Please review https://github.com/meshcloud/collie-cli/#prerequisites for installation instructions".` + `"${cmd}" cli is not installed in a supported version. Please review https://github.com/meshcloud/collie-cli/#prerequisites for installation instructions".`, ); case PlatformCommandInstallationStatus.Installed: break; } } - private buildShellRunner(options: CmdGlobalOptions) { + private buildShellRunner( + options: CmdGlobalOptions, + env?: Record, + ) { let shellRunner = new ShellRunner(); if (options.verbose) { @@ -103,6 +108,11 @@ export class CliApiFacadeFactory { } else if (isatty && !isWindows) { shellRunner = new LoaderShellRunner(shellRunner, new TTY()); } + + if (env) { + shellRunner = new DefaultEnvShellRunner(shellRunner, env); + } + return shellRunner; } } diff --git a/src/commands/foundation/new.command.ts b/src/commands/foundation/new.command.ts index 63adc892..d5e1fbc8 100644 --- a/src/commands/foundation/new.command.ts +++ b/src/commands/foundation/new.command.ts @@ -33,8 +33,6 @@ export function registerNewCmd(program: Command) { const foundationPath = repo.resolvePath("foundations", foundation); - // todo: this is stupidly hardcoded for now, would need some more dynamic detection of platforms and also - // possibly prompt the user for "do you want to add A? cool, missing a param there, what's your X?" etc. const factory = new CliApiFacadeFactory(logger, opts); const platformEntries = await promptPlatformEntries(logger, factory); @@ -55,7 +53,7 @@ export function registerNewCmd(program: Command) { await dir.write(d, ""); logger.progress( - "generated new foundation " + repo.relativePath(foundationPath) + "generated new foundation " + repo.relativePath(foundationPath), ); }); } @@ -74,11 +72,11 @@ Welcome to your cloud foundation. async function detectPlatform( logger: Logger, platform: string, - builder: () => Promise + builder: () => Promise, ): Promise { try { console.log( - `searching for a valid ${platform} platform in your current environment...` + `searching for a valid ${platform} platform in your current environment...`, ); const dir = await builder(); @@ -97,8 +95,11 @@ async function detectPlatform( async function promptPlatformEntries( logger: Logger, - factory: CliApiFacadeFactory + factory: CliApiFacadeFactory, ) { + // todo: this is stupidly hardcoded for now, would need some more dynamic detection of platforms and also + // possibly prompt the user for "do you want to add A? cool, missing a param there, what's your X?" etc. + const entries: Dir[] = [ await detectPlatform(logger, "Azure", () => setupAzurePlatform(factory)), await detectPlatform(logger, "AWS", () => setupAwsPlatform(factory)), @@ -126,7 +127,7 @@ function generateAwsReadmeMd(identity: CallerIdentity): string { name: "aws", aws: { accountId: identity.Account, - accountAccessRole: "OrganizationAccountAccessRole" // todo: be more smart about this default + accountAccessRole: "OrganizationAccountAccessRole", // todo: be more smart about this default }, cli: { aws: { @@ -153,7 +154,7 @@ async function setupGcpPlatform(factory: CliApiFacadeFactory): Promise { if (!project) { throw new MeshError( - "'gcloud config list' does not have a configured project" + "'gcloud config list' does not have a configured project", ); } @@ -198,6 +199,8 @@ async function setupAzurePlatform(factory: CliApiFacadeFactory): Promise { } function generateAzureReadmeMd(account: Account): string { + const configDir = Deno.env.get("AZURE_CONFIG_DIR"); + const frontmatter: PlatformConfigAzure = { name: "azure", azure: { @@ -206,7 +209,7 @@ function generateAzureReadmeMd(account: Account): string { }, cli: { az: { - AZURE_CONFIG_DIR: Deno.env.get("AZURE_CONFIG_DIR") || "", + ...(configDir && { AZURE_CONFIG_DIR: configDir }), }, }, }; diff --git a/src/commands/tenant/list.command.ts b/src/commands/tenant/list.command.ts index 5f3af6cb..62a59c36 100644 --- a/src/commands/tenant/list.command.ts +++ b/src/commands/tenant/list.command.ts @@ -9,42 +9,56 @@ import { CmdGlobalOptions, OutputFormatType } from "../cmd-options.ts"; import { isatty } from "../tty.ts"; import { TenantListPresenterFactory } from "../../presentation/tenant-list-presenter-factory.ts"; import { CollieRepository } from "/model/CollieRepository.ts"; +import { FoundationRepository } from "../../model/FoundationRepository.ts"; +import { MeshFoundationAdapterFactory } from "../../mesh/MeshFoundationAdapterFactory.ts"; +import { Logger } from "../../cli/Logger.ts"; +import { CliApiFacadeFactory } from "../../api/CliApiFacadeFactory.ts"; export function registerListCommand(program: Command) { - const listTenants = new Command() + program + .command("list ") // type must be added on every level that uses this type. Maybe bug in Cliffy? .type("output", OutputFormatType) .description( "Returns a list of tenants with their name, id, tags and platform.", ) .action(listTenantAction); - - program.command("list", listTenants); } -export async function listTenantAction(options: CmdGlobalOptions) { - const repo = await CollieRepository.load("./"); +export async function listTenantAction(options: CmdGlobalOptions, foundation: string) { + const collieRepo = await CollieRepository.load("./"); + const logger = new Logger(collieRepo, options); await setupLogger(options); - await verifyCliAvailability(); - const config = loadConfig(); - const meshAdapterFactory = new MeshAdapterFactory(config); + // todo: unify logging infra + + const foundationRepo = await FoundationRepository.load( + collieRepo, + foundation, + ); + + const facadeFactory = new CliApiFacadeFactory(logger, options); + const meshAdapterFactory = new MeshFoundationAdapterFactory( + collieRepo, + foundationRepo, + facadeFactory, + ); + const queryStatistics = new QueryStatistics(); - const meshAdapter = meshAdapterFactory.buildMeshAdapter( - options, + const meshAdapter = await meshAdapterFactory.buildMeshAdapter( queryStatistics, ); const allTenants = await meshAdapter.getMeshTenants(); const tableFactory = new MeshTableFactory(isatty); - const presenterFactory = new TenantListPresenterFactory(tableFactory); const presenter = presenterFactory.buildPresenter( options.output, allTenants, queryStatistics, ); + presenter.present(); } diff --git a/src/mesh/MeshFoundationAdapterFactory.ts b/src/mesh/MeshFoundationAdapterFactory.ts new file mode 100644 index 00000000..e5d7e7e2 --- /dev/null +++ b/src/mesh/MeshFoundationAdapterFactory.ts @@ -0,0 +1,121 @@ +import { AwsMeshAdapter } from "/api/aws/aws-mesh-adapter.ts"; +import { AzureMeshAdapter } from "/api/az/azure-mesh-adapter.ts"; +import { MultiMeshAdapter } from "./multi-mesh-adapter.ts"; +import { GcpMeshAdapter } from "/api/gcloud/gcp-mesh-adapter.ts"; +import { MeshAdapter } from "./mesh-adapter.ts"; +import { TimeWindowCalculator } from "./time-window-calculator.ts"; +import { newMeshTenantRepository } from "../db/mesh-tenant-repository.ts"; +import { CachingMeshAdapterDecorator } from "./caching-mesh-adapter-decorator.ts"; +import { StatsMeshAdapterDecorator } from "./stats-mesh-adapter-decorator.ts"; +import { MeshPlatform } from "./mesh-tenant.model.ts"; +import { + QueryStatistics, + STATS_LAYER_CACHE, + STATS_LAYER_PLATFORM, +} from "./query-statistics.ts"; +import { MeshError } from "../errors.ts"; +import { MeshTenantChangeDetector } from "./mesh-tenant-change-detector.ts"; +import { FoundationRepository } from "../model/FoundationRepository.ts"; +import { PlatformConfig } from "../model/PlatformConfig.ts"; +import { CollieRepository } from "../model/CollieRepository.ts"; +import { CliApiFacadeFactory } from "../api/CliApiFacadeFactory.ts"; + +/** + * Should consume the cli configuration in order to build the + * proper adapter. + */ +export class MeshFoundationAdapterFactory { + private readonly timeWindowCalc = new TimeWindowCalculator(); + private readonly tenantChangeDetector = new MeshTenantChangeDetector(); + + constructor( + private readonly collie: CollieRepository, + private readonly foundation: FoundationRepository, + private readonly facadeFactory: CliApiFacadeFactory, + ) {} + + async buildMeshAdapter(queryStats: QueryStatistics): Promise { + const buildAdapterTasks = this.foundation.platforms.map((x) => + this.buildPlatformAdapter(x, queryStats) + ); + + const adapters = await Promise.all(buildAdapterTasks); + + // TODO: change this to cache per platform! + + // There are multiple ways of doing it. Currently we only fetch everything or nothing, which is easier I guess. + // we could also split every platform into an individual repository folder and perform fetching and caching on a per-platform + // basis. For now we go with the easier part. + const tenantRepository = newMeshTenantRepository(); + + const cachingMeshAdapter = new CachingMeshAdapterDecorator( + tenantRepository, + new MultiMeshAdapter(adapters), + ); + + if (queryStats) { + return new StatsMeshAdapterDecorator( + cachingMeshAdapter, + "cache", + STATS_LAYER_CACHE, + queryStats, + ); + } + + return cachingMeshAdapter; + } + + // TODO: caching per platform, optimize stats decorator building + async buildPlatformAdapter( + config: PlatformConfig, + queryStats: QueryStatistics, + ): Promise { + if ("aws" in config) { + const aws = await this.facadeFactory.buildAws(config.cli.aws); + const awsAdapter = new AwsMeshAdapter( + aws, + config.aws.accountAccessRole, + this.tenantChangeDetector, + ); + + return new StatsMeshAdapterDecorator( + awsAdapter, + MeshPlatform.AWS, // todo: needs to be per platform instance, not per platform type + STATS_LAYER_PLATFORM, + queryStats, + ); + } else if ("azure" in config) { + const az = await this.facadeFactory.buildAz(config.cli.az); + const azureAdapter = new AzureMeshAdapter(az, this.tenantChangeDetector); + + return new StatsMeshAdapterDecorator( + azureAdapter, + MeshPlatform.Azure, + STATS_LAYER_PLATFORM, + queryStats, + ); + } else if ("gcp" in config) { + const gcloud = await this.facadeFactory.buildGcloud(config.cli.gcloud); + const gcpAdapter = new GcpMeshAdapter( + gcloud, + this.timeWindowCalc, + this.tenantChangeDetector, + ); + + return new StatsMeshAdapterDecorator( + gcpAdapter, + MeshPlatform.GCP, + STATS_LAYER_PLATFORM, + queryStats, + ); + } else { + throw new MeshError( + `Invalid platform definition with unknown platform type at ${ + this.collie.relativePath( + this.foundation.resolvePlatformPath(config), + ) + }`, + ); + } + } +} diff --git a/src/mesh/query-statistics.ts b/src/mesh/query-statistics.ts index 8911f980..0430c75b 100644 --- a/src/mesh/query-statistics.ts +++ b/src/mesh/query-statistics.ts @@ -2,6 +2,9 @@ import { MeshPlatform } from "./mesh-tenant.model.ts"; type DurationContainer = { [P in MeshPlatform | "cache"]?: number }; +export const STATS_LAYER_CACHE = 0; +export const STATS_LAYER_PLATFORM = 1; + export class QueryStatistics { duration: DurationContainer = {}; diff --git a/src/model/CliToolEnv.ts b/src/model/CliToolEnv.ts index eb4dadb2..2ea63476 100644 --- a/src/model/CliToolEnv.ts +++ b/src/model/CliToolEnv.ts @@ -1,19 +1,21 @@ -export interface AzCliEnv { +// these need to be types due to https://github.com/microsoft/TypeScript/issues/15300 + +export type AzCliEnv = { /** * see https://stackoverflow.com/questions/33137145/can-i-access-multiple-azure-accounts-with-azure-cli-from-the-same-machine-at-sam * todo: this works, but requires interactive login into the right tenant */ - AZURE_CONFIG_DIR: string; -} + AZURE_CONFIG_DIR?: string; +}; -export interface GcloudCliEnv { +export type GcloudCliEnv = { CLOUDSDK_ACTIVE_CONFIG_NAME: string; -} +}; -export interface AwsCliEnv { +export type AwsCliEnv = { AWS_CONFIG_FILE?: string; AWS_PROFILE: string; -} +}; export interface CliToolEnv { az?: AzCliEnv; diff --git a/src/model/FoundationRepository.ts b/src/model/FoundationRepository.ts new file mode 100644 index 00000000..8612e2c7 --- /dev/null +++ b/src/model/FoundationRepository.ts @@ -0,0 +1,124 @@ +import * as fs from "std/fs"; +import * as path from "std/path"; +import { + FoundationConfig, + MeshStackConfig, +} from "../model/FoundationConfig.ts"; +import { CollieRepository } from "./CollieRepository.ts"; +import { MarkdownDocument } from "./MarkdownDocument.ts"; +import { PlatformConfig } from "./PlatformConfig.ts"; + +export class FoundationRepository { + constructor( + private readonly foundationDir: string, + private readonly config: FoundationConfig, + ) {} + + public get name(): string { + return this.config.name; + } + + public get platforms(): PlatformConfig[] { + return this.config.platforms; + } + + findPlatform(platform: string) { + const p = this.config.platforms.find((x) => x.name === platform); + if (!p) { + throw new Error( + `Could not find platform named "${platform}" in configuration.`, + ); + } + + return p; + } + + /** + * Resolve a path relative to the foundation + */ + resolvePath(...pathSegments: string[]) { + return path.resolve(this.foundationDir, ...pathSegments); + } + + /** + * Resolve a path relative to a platform + */ + resolvePlatformPath(platform: PlatformConfig, ...pathSegments: string[]) { + return this.resolvePath("platforms", platform.name, ...pathSegments); + } + + static async load( + kit: CollieRepository, + foundation: string, + ): Promise { + const foundationDir = kit.resolvePath("foundations", foundation); + + const foundationReadme = await FoundationRepository.parseFoundationReadme( + kit, + foundationDir, + ); + + const platforms = await FoundationRepository.parsePlatformReadmes( + kit, + foundationDir, + ); + + const config: FoundationConfig = { + name: foundation, + meshStack: foundationReadme.frontmatter.meshStack, + platforms, + }; + + return new FoundationRepository(foundationDir, config); + } + + private static async parseFoundationReadme( + kit: CollieRepository, + foundationDir: string, + ) { + const readmePath = path.join(foundationDir, "README.md"); + const text = await Deno.readTextFile(readmePath); + const md = await MarkdownDocument.parse(text); + + if (!md) { + throw new Error( + "Failed to parse foundation README at " + kit.relativePath(readmePath), + ); + } + + return md; + } + + private static async parsePlatformReadmes( + kit: CollieRepository, + foundationDir: string, + ): Promise { + const platforms: PlatformConfig[] = []; + + for await ( + const file of fs.expandGlob("platforms/*/README.md", { + root: foundationDir, + }) + ) { + const text = await Deno.readTextFile(file.path); + const md = await MarkdownDocument.parse(text); + + // todo: validate config is valid? + const config = md?.frontmatter; + if (!config) { + throw new Error( + "Failed to parse foundation README at " + kit.relativePath(file.path), + ); + } + + platforms.push(config as PlatformConfig); + } + + return platforms; + } +} + +interface FoundationFrontmatter { + name: string; + meshStack: MeshStackConfig; +} diff --git a/src/process/DefaultEnvShellRunner.ts b/src/process/DefaultEnvShellRunner.ts new file mode 100644 index 00000000..0ed3f078 --- /dev/null +++ b/src/process/DefaultEnvShellRunner.ts @@ -0,0 +1,16 @@ +import { ShellOutput } from "./shell-output.ts"; +import { IShellRunner } from "./shell-runner.interface.ts"; + +export class DefaultEnvShellRunner implements IShellRunner { + constructor( + private readonly runner: IShellRunner, + private readonly defaultEnv: Record, + ) {} + + public async run( + commandStr: string, + env?: Record, + ): Promise { + return await this.runner.run(commandStr, { ...this.defaultEnv, ...env }); + } +} diff --git a/src/process/shell-runner.ts b/src/process/shell-runner.ts index 5b25aae9..2fc319b9 100644 --- a/src/process/shell-runner.ts +++ b/src/process/shell-runner.ts @@ -28,12 +28,18 @@ export class ShellRunner implements IShellRunner { const rawError = await p.stderrOutput(); const { code } = await p.status(); - console.debug(`Exit code for running '${commandStr}' is ${code}`); - - return { + const result = { code: code, stderr: decoder.decode(rawError), stdout: decoder.decode(rawOutput), }; + + console.debug(`Exit code for running '${commandStr}' is ${code}`); + if (code != 0) { + console.debug(result.stdout); + console.debug(result.stderr); + } + + return result; } }