From 5adc04e26dbdfae7b4da5f9484e4d41116d73fd6 Mon Sep 17 00:00:00 2001 From: Iulia Manda Date: Fri, 22 Dec 2023 16:20:12 +0100 Subject: [PATCH] fix: correctky check if sysdig is enabled --- src/data-scraper/index.ts | 95 ----------------- src/data-scraper/scraping-v1.ts | 112 +++++++++++++++++++++ src/healthcheck.ts | 3 +- src/index.ts | 12 ++- test/unit/data-scraper/scrape-data.spec.ts | 3 +- 5 files changed, 125 insertions(+), 100 deletions(-) create mode 100644 src/data-scraper/scraping-v1.ts diff --git a/src/data-scraper/index.ts b/src/data-scraper/index.ts index fe2c293f2..526330990 100644 --- a/src/data-scraper/index.ts +++ b/src/data-scraper/index.ts @@ -13,14 +13,6 @@ const httpsAgent = new HttpsAgent({ rejectUnauthorized: false, }); -// Soon to be deprecated -function getSysdigV1Url(): string { - return config.SYSDIG_ENDPOINT + '/v1/runtimeimages'; -} -function getSysdigV1AuthHeader(): string { - return `Bearer ${config.SYSDIG_TOKEN}`; -} - function getSysdigUrl(): string { return ( 'https://' + @@ -128,90 +120,3 @@ export async function scrapeData(): Promise { } } } - -/** NOTE: This function can throw, so the caller should handle errors. */ -export async function validateConnectivityV1(): Promise { - const url = getSysdigV1Url(); - const header = getSysdigV1AuthHeader(); - const reqOptions: NeedleOptions = { - agent: httpsAgent, - headers: { - authorization: header, - }, - timeout: 10_000, - }; - - const limit: number = 1; - const cursor: string = ''; - const { response } = await retryRequest( - 'get', - `${url}?limit=${limit}&cursor=${cursor}`, - {}, - reqOptions, - ); - if (!isSuccessStatusCode(response.statusCode)) { - throw new Error(`${response.statusCode} ${response.statusMessage}`); - } -} - -export async function scrapeDataV1(): Promise { - const url = getSysdigV1Url(); - const header = getSysdigV1AuthHeader(); - - // limit: min 1, max 500, default 250 - const limit: number = 10; - const reqOptions: NeedleOptions = { - agent: httpsAgent, - headers: { - authorization: header, - }, - }; - - let cursor: string = ''; - while (true) { - try { - logger.info({ cursor }, 'attempting to get runtime images'); - - const { response, attempt } = await retryRequest( - 'get', - `${url}?limit=${limit}&cursor=${cursor}`, - {}, - reqOptions, - ); - if (!isSuccessStatusCode(response.statusCode)) { - throw new Error(`${response.statusCode} ${response.statusMessage}`); - } - - logger.info( - { - attempt, - cursor, - }, - 'runtime images received successfully', - ); - - const responseBody: IRuntimeImagesResponse | undefined = response.body; - const runtimeDataPayload = constructRuntimeData( - responseBody?.data ?? [], - 1, - ); - - logger.info({}, 'sending runtime data upstream'); - await sendRuntimeData(runtimeDataPayload); - - cursor = responseBody?.page.next || ''; - if (!cursor) { - break; - } - } catch (error) { - logger.error( - { - error, - cursor, - }, - 'could not get runtime images', - ); - break; - } - } -} diff --git a/src/data-scraper/scraping-v1.ts b/src/data-scraper/scraping-v1.ts new file mode 100644 index 000000000..bda1872a8 --- /dev/null +++ b/src/data-scraper/scraping-v1.ts @@ -0,0 +1,112 @@ +import { logger } from '../common/logger'; +import { config } from '../common/config'; +import { sendRuntimeData } from '../transmitter'; +import { constructRuntimeData } from '../transmitter/payload'; +import { retryRequest } from '../transmitter'; +import { IRuntimeImagesResponse } from '../transmitter/types'; +import { NeedleOptions } from 'needle'; +import { Agent as HttpsAgent } from 'https'; + +const httpsAgent = new HttpsAgent({ + keepAlive: true, + // We agreed with Sysdig to skip TLS certificates validation for HTTPS connection. + rejectUnauthorized: false, +}); + +function getSysdigV1Url(): string { + return config.SYSDIG_ENDPOINT + '/v1/runtimeimages'; +} +function getSysdigV1AuthHeader(): string { + return `Bearer ${config.SYSDIG_TOKEN}`; +} + +function isSuccessStatusCode(statusCode: number | undefined): boolean { + return statusCode !== undefined && statusCode >= 200 && statusCode < 300; +} + +/** NOTE: This function can throw, so the caller should handle errors. */ +export async function validateConnectivityV1(): Promise { + const url = getSysdigV1Url(); + const header = getSysdigV1AuthHeader(); + const reqOptions: NeedleOptions = { + agent: httpsAgent, + headers: { + authorization: header, + }, + timeout: 10_000, + }; + + const limit: number = 1; + const cursor: string = ''; + const { response } = await retryRequest( + 'get', + `${url}?limit=${limit}&cursor=${cursor}`, + {}, + reqOptions, + ); + if (!isSuccessStatusCode(response.statusCode)) { + throw new Error(`${response.statusCode} ${response.statusMessage}`); + } +} + +export async function scrapeDataV1(): Promise { + const url = getSysdigV1Url(); + const header = getSysdigV1AuthHeader(); + + // limit: min 1, max 500, default 250 + const limit: number = 10; + const reqOptions: NeedleOptions = { + agent: httpsAgent, + headers: { + authorization: header, + }, + }; + + let cursor: string = ''; + while (true) { + try { + logger.info({ cursor }, 'attempting to get runtime images'); + + const { response, attempt } = await retryRequest( + 'get', + `${url}?limit=${limit}&cursor=${cursor}`, + {}, + reqOptions, + ); + if (!isSuccessStatusCode(response.statusCode)) { + throw new Error(`${response.statusCode} ${response.statusMessage}`); + } + + logger.info( + { + attempt, + cursor, + }, + 'runtime images received successfully', + ); + + const responseBody: IRuntimeImagesResponse | undefined = response.body; + const runtimeDataPayload = constructRuntimeData( + responseBody?.data ?? [], + 1, + ); + + logger.info({}, 'sending runtime data upstream'); + await sendRuntimeData(runtimeDataPayload); + + cursor = responseBody?.page.next || ''; + if (!cursor) { + break; + } + } catch (error) { + logger.error( + { + error, + cursor, + }, + 'could not get runtime images', + ); + break; + } + } +} diff --git a/src/healthcheck.ts b/src/healthcheck.ts index a0845cdf1..ad9069985 100644 --- a/src/healthcheck.ts +++ b/src/healthcheck.ts @@ -1,6 +1,7 @@ import { config } from './common/config'; import { logger } from './common/logger'; import { state } from './state'; +import { validateConnectivityV1 } from './data-scraper/scraping-v1'; import * as dataScraper from './data-scraper'; @@ -52,7 +53,7 @@ async function sysdigHealthCheck(): Promise { try { let sysdigVersion = getSysdigVersion(); if (sysdigVersion == sysdigV1) { - await dataScraper.validateConnectivityV1(); + await validateConnectivityV1(); } else { await dataScraper.validateConnectivity(); } diff --git a/src/index.ts b/src/index.ts index a24455796..470055638 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,7 +10,8 @@ import { beginWatchingWorkloads } from './supervisor/watchers'; import { loadAndSendWorkloadEventsPolicy } from './common/policy'; import { sendClusterMetadata } from './transmitter'; import { setSnykMonitorAgentId } from './supervisor/agent'; -import { scrapeData, scrapeDataV1 } from './data-scraper'; +import { scrapeData } from './data-scraper'; +import { scrapeDataV1 } from './data-scraper/scraping-v1'; import { getSysdigVersion, setupHealthCheck, sysdigV1 } from './healthcheck'; process.on('uncaughtException', (error) => { @@ -73,9 +74,14 @@ async function setupSysdigIntegration(): Promise { config.SYSDIG_REGION_URL && config.SYSDIG_RISK_SPOTLIGHT_TOKEN && config.SYSDIG_CLUSTER_NAME - ) || + ) && !(config.SYSDIG_ENDPOINT && config.SYSDIG_TOKEN) ) { + console.log( + config.SYSDIG_REGION_URL, + config.SYSDIG_RISK_SPOTLIGHT_TOKEN, + config.SYSDIG_CLUSTER_NAME, + ); console.log('Sysdig integration not enabled'); return; } @@ -83,7 +89,7 @@ async function setupSysdigIntegration(): Promise { let sysdigVersion = getSysdigVersion(); logger.info({}, `Sysdig ${sysdigVersion} data scraping starting`); - const initialInterval: number = 60 * 1000; // 20 mins in milliseconds + const initialInterval: number = 20 * 60 * 1000; // 20 mins in milliseconds setTimeout(async () => { try { if (sysdigVersion == sysdigV1) { diff --git a/test/unit/data-scraper/scrape-data.spec.ts b/test/unit/data-scraper/scrape-data.spec.ts index ad180a11b..d024deeab 100644 --- a/test/unit/data-scraper/scrape-data.spec.ts +++ b/test/unit/data-scraper/scrape-data.spec.ts @@ -1,7 +1,8 @@ import nock from 'nock'; import { config } from '../../../src/common/config'; -import { scrapeData, scrapeDataV1 } from '../../../src/data-scraper'; +import { scrapeData } from '../../../src/data-scraper'; +import { scrapeDataV1 } from '../../../src/data-scraper/scraping-v1'; import * as transmitterTypes from '../../../src/transmitter/types';