From d319866538e190666b7640f92f787bafecf96a16 Mon Sep 17 00:00:00 2001 From: Irwansyah Date: Fri, 19 Jul 2024 14:17:04 +0700 Subject: [PATCH 01/10] chore: replace axios dependencies in insomnia.ts with http-client --- src/components/config/parser/insomnia.ts | 4 +-- src/components/http-client/index.ts | 46 ++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 src/components/http-client/index.ts diff --git a/src/components/config/parser/insomnia.ts b/src/components/config/parser/insomnia.ts index f901091e5..4e393be77 100644 --- a/src/components/config/parser/insomnia.ts +++ b/src/components/config/parser/insomnia.ts @@ -25,7 +25,7 @@ import type { Config } from '../../../interfaces/config' import yml from 'js-yaml' import { compile as compileTemplate } from 'handlebars' -import type { AxiosRequestHeaders, Method } from 'axios' +import { Method, HttpClientHeaders } from '../../http-client' import Joi from 'joi' const envValidator = Joi.object({ @@ -145,7 +145,7 @@ function mapInsomniaRequestToConfig(req: unknown) { // eslint-disable-next-line camelcase const url = compileTemplate(res.url)({ base_url: baseUrl }) const authorization = getAuthorizationHeader(res) - let headers: AxiosRequestHeaders | undefined + let headers: HttpClientHeaders | undefined if (authorization) headers = { authorization, diff --git a/src/components/http-client/index.ts b/src/components/http-client/index.ts new file mode 100644 index 000000000..f4fa6ba34 --- /dev/null +++ b/src/components/http-client/index.ts @@ -0,0 +1,46 @@ +/********************************************************************************** + * MIT License * + * * + * Copyright (c) 2021 Hyperjump Technology * + * * + * Permission is hereby granted, free of charge, to any person obtaining a copy * + * of this software and associated documentation files (the "Software"), to deal * + * in the Software without restriction, including without limitation the rights * + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * + * copies of the Software, and to permit persons to whom the Software is * + * furnished to do so, subject to the following conditions: * + * * + * The above copyright notice and this permission notice shall be included in all * + * copies or substantial portions of the Software. * + * * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * + * SOFTWARE. * + **********************************************************************************/ + +export type HttpClientHeaders = Record +export type Method = + | 'get' + | 'GET' + | 'delete' + | 'DELETE' + | 'head' + | 'HEAD' + | 'options' + | 'OPTIONS' + | 'post' + | 'POST' + | 'put' + | 'PUT' + | 'patch' + | 'PATCH' + | 'purge' + | 'PURGE' + | 'link' + | 'LINK' + | 'unlink' + | 'UNLINK' From 1b822056ee9dfbd0c91b7182ab590f7ee12ffc7a Mon Sep 17 00:00:00 2001 From: Irwansyah Date: Fri, 19 Jul 2024 14:17:04 +0700 Subject: [PATCH 02/10] chore: replace axios with fetch in probe related functions --- src/components/config/index.ts | 4 +- src/components/config/parser/insomnia.ts | 6 +- src/components/config/parser/parse.ts | 16 +- src/components/http-client/index.ts | 157 +++++++++++++++++--- src/components/probe/prober/http/request.ts | 88 +++-------- src/interfaces/http-client.ts | 88 +++++++++++ src/plugins/updater/index.ts | 13 +- src/utils/http.ts | 110 ++++---------- src/utils/public-ip.ts | 2 +- 9 files changed, 300 insertions(+), 184 deletions(-) create mode 100644 src/interfaces/http-client.ts diff --git a/src/components/config/index.ts b/src/components/config/index.ts index 2d935037f..8c841b90e 100644 --- a/src/components/config/index.ts +++ b/src/components/config/index.ts @@ -62,14 +62,14 @@ async function createExampleConfigFile() { try { const resp = await sendHttpRequest({ url }) - await writeFile(outputFilePath, resp.data, { encoding: 'utf8' }) + await writeFile(outputFilePath, resp.data as string, { encoding: 'utf8' }) } catch { const ymlConfig = ` probes: - id: '1' requests: - url: http://example.com - + db_limit: max_db_size: 1000000000 deleted_data: 1 diff --git a/src/components/config/parser/insomnia.ts b/src/components/config/parser/insomnia.ts index 4e393be77..5a2f9cf46 100644 --- a/src/components/config/parser/insomnia.ts +++ b/src/components/config/parser/insomnia.ts @@ -25,8 +25,9 @@ import type { Config } from '../../../interfaces/config' import yml from 'js-yaml' import { compile as compileTemplate } from 'handlebars' -import { Method, HttpClientHeaders } from '../../http-client' + import Joi from 'joi' +import { Method } from '../../../interfaces/http-client' const envValidator = Joi.object({ scheme: Joi.array().items(Joi.string()), @@ -145,13 +146,14 @@ function mapInsomniaRequestToConfig(req: unknown) { // eslint-disable-next-line camelcase const url = compileTemplate(res.url)({ base_url: baseUrl }) const authorization = getAuthorizationHeader(res) - let headers: HttpClientHeaders | undefined + let headers: { [key: string]: unknown } | undefined if (authorization) headers = { authorization, } if (res.headers) { if (headers === undefined) headers = {} + for (const h of res.headers) { headers[h.name] = h.value } diff --git a/src/components/config/parser/parse.ts b/src/components/config/parser/parse.ts index 29f18e00d..af8e8ca90 100644 --- a/src/components/config/parser/parse.ts +++ b/src/components/config/parser/parse.ts @@ -50,7 +50,7 @@ export async function parseByType( const config = isUrl(source) ? await getConfigFileFromUrl(source) : await readFile(source, { encoding: 'utf8' }) - const isEmpty = config.length === 0 + const isEmpty = (config as string).length === 0 if (isEmpty) { throw new Error(`Failed to read ${source}, the file is empty.`) @@ -58,15 +58,15 @@ export async function parseByType( const extension = path.extname(source) - if (type === 'har') return parseHarFile(config) - if (type === 'text') return parseConfigFromText(config) - if (type === 'postman') return parseConfigFromPostman(config) - if (type === 'sitemap') return parseConfigFromSitemap(config) + if (type === 'har') return parseHarFile(config as string) + if (type === 'text') return parseConfigFromText(config as string) + if (type === 'postman') return parseConfigFromPostman(config as string) + if (type === 'sitemap') return parseConfigFromSitemap(config as string) if (type === 'insomnia') - return parseInsomnia(config, extension.replace('.', '')) + return parseInsomnia(config as string, extension.replace('.', '')) return parseConfigByExt({ - config, + config: config as string, extension, source, }) @@ -74,7 +74,7 @@ export async function parseByType( async function getConfigFileFromUrl(url: string) { const config = await fetchConfigFile(url) - const isEmpty = config.length === 0 + const isEmpty = (config as string).length === 0 if (isEmpty) { throw new Error( diff --git a/src/components/http-client/index.ts b/src/components/http-client/index.ts index f4fa6ba34..d19ec22c5 100644 --- a/src/components/http-client/index.ts +++ b/src/components/http-client/index.ts @@ -22,25 +22,138 @@ * SOFTWARE. * **********************************************************************************/ -export type HttpClientHeaders = Record -export type Method = - | 'get' - | 'GET' - | 'delete' - | 'DELETE' - | 'head' - | 'HEAD' - | 'options' - | 'OPTIONS' - | 'post' - | 'POST' - | 'put' - | 'PUT' - | 'patch' - | 'PATCH' - | 'purge' - | 'PURGE' - | 'link' - | 'LINK' - | 'unlink' - | 'UNLINK' +import { ReadableStream } from 'node:stream/web' +import { + HttpClientHeaderList, + HttpClientHeaders, + HttpClientResponseType, + HttpClientRequestOptions, +} from '../../interfaces/http-client' +import { Agent, type HeadersInit } from 'undici' + +export class HttpClientResponse { + _fetchResponse: Response + _headers: HttpClientHeaderList | null = null + _isStreamResponseType: boolean + _data: unknown + + constructor( + fetchResponse: Response, + isStreamResponseType: boolean, + data?: unknown + ) { + this._fetchResponse = fetchResponse + this._isStreamResponseType = isStreamResponseType + this._data = data + } + + get headers(): HttpClientHeaders { + if (!this._headers) { + this._headers = new HttpClientHeaderList() + + for (const [k, v] of this._fetchResponse.headers.entries()) { + this._headers!.set(k, v) + } + } + + return this._headers + } + + get ok(): boolean { + return this._fetchResponse.ok + } + + get status(): number { + return this._fetchResponse.status + } + + get statusText(): string { + return this._fetchResponse.statusText + } + + get type(): HttpClientResponseType { + return this._fetchResponse.type + } + + // get url: string + // readonly redirected: boolean + + // readonly bodyUsed: boolean + + // readonly arrayBuffer: () => Promise + // readonly blob: () => Promise + // readonly formData: () => Promise + + json(): Promise { + return this._fetchResponse.json() + } + + text(): Promise { + return this._fetchResponse.text() + } + + // static error (): Response; + + get data(): ReadableStream | unknown { + if (this._isStreamResponseType) { + const reader = this._fetchResponse.body?.getReader() + return new ReadableStream({ + start(controller) { + return pump() + + function pump() { + if (!reader) return + + return reader + .read() + .then(({ done, value }: { done: any; value?: any }): any => { + // When no more data needs to be consumed, close the stream + if (done) { + controller.close() + return + } + + // Enqueue the next data chunk into our target stream + controller.enqueue(value) + return pump() + }) + } + }, + }) + } + + return this._data + } +} + +export const httpClient = async ( + url: string, + requestOptions: HttpClientRequestOptions, + isStreamResponseType: boolean = false +): Promise => { + const fetchResponse = await fetch(url, { + body: requestOptions.body === '' ? undefined : requestOptions.body, + redirect: requestOptions.redirect, + dispatcher: + requestOptions.allowUnauthorizedSsl === undefined + ? undefined + : new Agent({ + connect: { + rejectUnauthorized: !requestOptions.allowUnauthorizedSsl, + keepAlive: requestOptions.keepAlive, + }, + }), + + headers: requestOptions.headers as HeadersInit, + keepalive: requestOptions.keepAlive, + method: requestOptions.method, + signal: requestOptions.signal, + }) + + let data + if (!isStreamResponseType) { + data = await fetchResponse.json() + } + + return new HttpClientResponse(fetchResponse, isStreamResponseType, data) +} diff --git a/src/components/probe/prober/http/request.ts b/src/components/probe/prober/http/request.ts index ec1a086a5..a52fb81f0 100644 --- a/src/components/probe/prober/http/request.ts +++ b/src/components/probe/prober/http/request.ts @@ -37,10 +37,15 @@ import { import { getContext } from '../../../../context' import { icmpRequest } from '../icmp/request' import registerFakes from '../../../../utils/fakes' -import { sendHttpRequest, sendHttpRequestFetch } from '../../../../utils/http' +import { sendHttpRequest } from '../../../../utils/http' import { log } from '../../../../utils/pino' import { AxiosError } from 'axios' import { getErrorMessage } from '../../../../utils/catch-error-handler' +import { + HttpClientHeaderList, + HttpClientHeaders, + HttpClientBody, +} from '../../../../interfaces/http-client' // Register Handlebars helpers registerFakes(Handlebars) @@ -98,27 +103,19 @@ export async function httpRequest({ return icmpRequest({ host: renderedURL }) } - const requestHeaders = new Headers() + const requestHeaders = new HttpClientHeaderList() for (const [key, value] of Object.entries(newReq.headers || {})) { requestHeaders.set(key, value) } // Do the request using compiled URL and compiled headers (if exists) - const response = await (getContext().flags['native-fetch'] - ? probeHttpFetch({ - startTime, - maxRedirects: followRedirects, - renderedURL, - requestParams: { ...newReq, headers: requestHeaders }, - allowUnauthorized, - }) - : probeHttpAxios({ - startTime, - maxRedirects: followRedirects, - renderedURL, - requestParams: { ...newReq, headers: requestHeaders }, - allowUnauthorized, - })) + const response = await probeHttpClient({ + startTime, + maxRedirects: followRedirects, + renderedURL, + requestParams: { ...newReq, headers: requestHeaders }, + allowUnauthorized, + }) return response } catch (error: unknown) { @@ -265,7 +262,7 @@ function compileBody({ return { headers: newHeaders, body: newBody } } -async function probeHttpFetch({ +async function probeHttpClient({ startTime, renderedURL, requestParams, @@ -278,9 +275,9 @@ async function probeHttpFetch({ maxRedirects: number requestParams: { method: string | undefined - headers: Headers | undefined + headers: HttpClientHeaders | undefined timeout: number - body?: BodyInit + body?: HttpClientBody ping: boolean | undefined } }): Promise { @@ -288,7 +285,7 @@ async function probeHttpFetch({ log.info(`Probing ${renderedURL} with Node.js fetch`) } - const response = await sendHttpRequestFetch({ + const response = await sendHttpRequest({ ...requestParams, allowUnauthorizedSsl: allowUnauthorized, keepalive: true, @@ -309,9 +306,9 @@ async function probeHttpFetch({ } } - const responseBody = response.headers - .get('Content-Type') - ?.includes('application/json') + const responseBody = ( + (response.headers as HttpClientHeaderList).get('Content-Type') as string + )?.includes('application/json') ? await response.json() : await response.text() @@ -326,49 +323,6 @@ async function probeHttpFetch({ } } -type ProbeHTTPAxiosParams = { - startTime: number - renderedURL: string - allowUnauthorized: boolean | undefined - maxRedirects: number - requestParams: { - method: string | undefined - headers: Headers | undefined - timeout: number - body?: BodyInit - ping: boolean | undefined - } -} - -async function probeHttpAxios({ - startTime, - renderedURL, - requestParams, - allowUnauthorized, - maxRedirects, -}: ProbeHTTPAxiosParams): Promise { - const resp = await sendHttpRequest({ - ...requestParams, - allowUnauthorizedSsl: allowUnauthorized, - keepalive: true, - url: renderedURL, - maxRedirects, - }) - - const responseTime = Date.now() - startTime - const { data, headers, status } = resp - - return { - requestType: 'HTTP', - data, - body: data, - status, - headers, - responseTime, - result: probeRequestResult.success, - } -} - export function generateRequestChainingBody( body: object | string, responses: ProbeRequestResponse[] diff --git a/src/interfaces/http-client.ts b/src/interfaces/http-client.ts new file mode 100644 index 000000000..67a2cab77 --- /dev/null +++ b/src/interfaces/http-client.ts @@ -0,0 +1,88 @@ +export type Method = + | 'get' + | 'GET' + | 'delete' + | 'DELETE' + | 'head' + | 'HEAD' + | 'options' + | 'OPTIONS' + | 'post' + | 'POST' + | 'put' + | 'PUT' + | 'patch' + | 'PATCH' + | 'purge' + | 'PURGE' + | 'link' + | 'LINK' + | 'unlink' + | 'UNLINK' + +export class HttpClientHeaderList extends Map< + string, + string | ReadonlyArray | number | boolean +> {} + +export type HttpClientRequestRedirect = 'error' | 'follow' | 'manual' +export type HttpClientHeaders = + | string[][] + | Record | number | boolean> + | HttpClientHeaderList + +export type HttpClientBody = + | ArrayBuffer + | AsyncIterable + | Blob + | FormData + | Iterable + | NodeJS.ArrayBufferView + | URLSearchParams + | null + | string + +export type HttpClientRequestCredentials = 'omit' | 'include' | 'same-origin' +export type HttpClientRequestMode = + | 'cors' + | 'navigate' + | 'no-cors' + | 'same-origin' +export type HttpClientReferrerPolicy = + | '' + | 'no-referrer' + | 'no-referrer-when-downgrade' + | 'origin' + | 'origin-when-cross-origin' + | 'same-origin' + | 'strict-origin' + | 'strict-origin-when-cross-origin' + | 'unsafe-url' + +export type HttpClientRequestDuplex = 'half' + +export interface HttpClientRequestOptions { + method?: string + keepalive?: boolean + headers?: HttpClientHeaders + body?: HttpClientBody + redirect?: HttpClientRequestRedirect + integrity?: string + signal?: AbortSignal + credentials?: HttpClientRequestCredentials + mode?: HttpClientRequestMode + referrer?: string + referrerPolicy?: HttpClientReferrerPolicy + window?: null + allowUnauthorizedSsl?: boolean + duplex?: HttpClientRequestDuplex + keepAlive?: boolean +} + +export type HttpClientResponseType = + | 'basic' + | 'cors' + | 'default' + | 'error' + | 'opaque' + | 'opaqueredirect' diff --git a/src/plugins/updater/index.ts b/src/plugins/updater/index.ts index 466ae5ab5..70aa07076 100644 --- a/src/plugins/updater/index.ts +++ b/src/plugins/updater/index.ts @@ -92,7 +92,9 @@ async function runUpdater(config: IConfig, updateMode: UpdateMode) { url: 'https://registry.npmjs.org/@hyperjumptech/monika', }) - const latestVersion = data['dist-tags'].latest + const latestVersion = ( + (data as Record)['dist-tags'] as Record + ).latest if (latestVersion === currentVersion || config.debug) { const nextCheck = new Date(Date.now() + DEFAULT_UPDATE_CHECK * 1000) const date = format(nextCheck, 'yyyy-MM-dd HH:mm:ss XXX') @@ -106,7 +108,7 @@ async function runUpdater(config: IConfig, updateMode: UpdateMode) { } const [currentMajor, currentMinor] = currentVersion.split('.') - const { time } = data + const { time } = data as Record // versions: key-value data with semver as key and timestamp as value // sorted descending by timestamp const versions = Object.keys(time) @@ -273,7 +275,7 @@ async function downloadMonika( responseType: 'stream', }) - const { data: checksum }: { data: string } = await sendHttpRequest({ + const { data: checksum }: { data: unknown } = await sendHttpRequest({ url: `https://github.com/hyperjumptech/monika/releases/download/v${remoteVersion}/${filename}-CHECKSUM.txt`, }) @@ -289,7 +291,10 @@ async function downloadMonika( writer.on('close', () => { log.info(`Updater: verifying download`) - const hashRemote = checksum.slice(0, checksum.indexOf(' ')) + const hashRemote = (checksum as string).slice( + 0, + (checksum as string).indexOf(' ') + ) const hashTarball = hasha.fromFileSync(targetPath, { algorithm: 'sha256', }) diff --git a/src/utils/http.ts b/src/utils/http.ts index af6d86f2b..e7ba6b7bf 100644 --- a/src/utils/http.ts +++ b/src/utils/http.ts @@ -22,117 +22,70 @@ * SOFTWARE. * **********************************************************************************/ -import type { AxiosRequestHeaders, AxiosResponse } from 'axios' -import axios from 'axios' -import http from 'node:http' -import https from 'node:https' -import { Agent, type HeadersInit } from 'undici' +import { httpClient, HttpClientResponse } from '../components/http-client' +import { + HttpClientHeaders, + HttpClientRequestOptions, + HttpClientHeaderList, +} from '../interfaces/http-client' type HttpRequestParams = { url: string maxRedirects?: number - headers?: HeadersInit + headers?: HttpClientHeaders timeout?: number allowUnauthorizedSsl?: boolean responseType?: 'stream' -} & Omit +} & Omit -// Keep the agents alive to reduce the overhead of DNS queries and creating TCP connection. -// More information here: https://rakshanshetty.in/nodejs-http-keep-alive/ -const httpAgent = new http.Agent({ keepAlive: true }) -const httpsAgent = new https.Agent({ keepAlive: true }) export const DEFAULT_TIMEOUT = 10_000 -// Create an instance of axios here so it will be reused instead of creating a new one all the time. -const axiosInstance = axios.create() - export async function sendHttpRequest( config: HttpRequestParams -): Promise { - const { allowUnauthorizedSsl, body, headers, timeout, ...options } = config - - return axiosInstance.request({ - ...options, - data: body, - headers: convertHeadersToAxios(headers), - timeout: timeout ?? DEFAULT_TIMEOUT, // Ensure default timeout if not filled. - httpAgent, - httpsAgent: allowUnauthorizedSsl - ? new https.Agent({ keepAlive: true, rejectUnauthorized: true }) - : httpsAgent, - }) -} - -function convertHeadersToAxios(headersInit: HeadersInit | undefined) { - const headers: AxiosRequestHeaders = {} - - if (headersInit instanceof Headers) { - // If headersInit is a Headers object - for (const [key, value] of headersInit.entries()) { - headers[key] = value - } - - return headers - } - - if (typeof headersInit === 'object') { - // If headersInit is a plain object - for (const [key, value] of Object.entries(headersInit)) { - headers[key] = value as never - } - - return headers - } - - return headers -} - -export async function sendHttpRequestFetch( - config: HttpRequestParams -): Promise { +): Promise { const { maxRedirects, timeout, url } = config const controller = new AbortController() const { signal } = controller const timeoutId = setTimeout(() => { controller.abort() }, timeout || DEFAULT_TIMEOUT) - const fetcher = compileFetch(config, signal) + const fetcher = compileHttpClient(config, signal) return fetchRedirect(url, maxRedirects, fetcher).then((response) => { clearTimeout(timeoutId) return response }) } -function compileFetch( +function compileHttpClient( config: HttpRequestParams, signal: AbortSignal -): (url: string) => Promise { - const { allowUnauthorizedSsl, body, headers, method } = config +): (url: string) => Promise { + const { allowUnauthorizedSsl, body, headers, method, responseType } = config return (url) => - fetch(url, { - body: body === '' ? undefined : body, - redirect: 'manual', - dispatcher: new Agent({ - connect: { - rejectUnauthorized: !allowUnauthorizedSsl, - }, - }), - headers, - keepalive: true, - method, - signal, - }) + httpClient( + url, + { + body: body === '' ? undefined : body, + redirect: 'manual', + allowUnauthorizedSsl, + headers, + keepalive: true, + method, + signal, + }, + responseType === 'stream' + ) } // fetchRedirect handles HTTP status code 3xx returned by fetcher async function fetchRedirect( url: string, maxRedirects: number | undefined, - fetcher: (url: string) => Promise + fetcher: (url: string) => Promise ) { let redirected = 0 - let currentResponse: Response + let currentResponse: HttpClientResponse let nextUrl = url // do HTTP fetch request at least once @@ -148,16 +101,17 @@ async function fetchRedirect( // location header could either be full url, relative path, or absolute path // e.g. "https://something.tld", "new/path", "/new/path", respectively // refer to : RFC-7231 https://datatracker.ietf.org/doc/html/rfc7231#section-7.1.2 - const newLocation = currentResponse.headers.get('location') || '' + const newLocation = + (currentResponse.headers as HttpClientHeaderList).get('location') || '' // try-catch to evaluate if redirect location is a url try { // when it is valid url, immediately set nextUrl from location header nextUrl = new URL( - currentResponse.headers.get('location') || '' + (currentResponse.headers as HttpClientHeaderList).get('location') || '' ).toString() } catch { // new redirect location is relative / absolute url - const newEndpoint = newLocation.startsWith('/') + const newEndpoint = (newLocation as string).startsWith('/') ? newLocation : `/${newLocation}` // parse nextUrl to Node.js URL to get protocol and host diff --git a/src/utils/public-ip.ts b/src/utils/public-ip.ts index 3c4799576..bdfc334b5 100644 --- a/src/utils/public-ip.ts +++ b/src/utils/public-ip.ts @@ -69,7 +69,7 @@ async function fetchPublicNetworkInfo(): Promise { const response = await sendHttpRequest({ url: `http://ip-api.com/json/${publicIp}`, }) - const { country, city, isp } = response.data + const { country, city, isp } = response.data as Record return { country, From 3df784328baa1c9b904e8f125003be7c5035cd64 Mon Sep 17 00:00:00 2001 From: Irwansyah Date: Fri, 19 Jul 2024 14:17:04 +0700 Subject: [PATCH 03/10] chore: replace axios with fetch in report-to-symon --- src/workers/report-to-symon.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/workers/report-to-symon.ts b/src/workers/report-to-symon.ts index 10a4c9907..bf25dfbba 100644 --- a/src/workers/report-to-symon.ts +++ b/src/workers/report-to-symon.ts @@ -22,7 +22,6 @@ * SOFTWARE. * **********************************************************************************/ -import axios from 'axios' import path from 'path' import { open } from 'sqlite' import { verbose } from 'sqlite3' @@ -35,6 +34,7 @@ import { getUnreportedLogs, } from '../components/logger/history' import { log } from '../utils/pino' +import { httpClient } from 'src/components/http-client' const dbPath = path.resolve(process.cwd(), 'monika-logs.db') export default async (stringifiedData: string) => { @@ -61,20 +61,19 @@ export default async (stringifiedData: string) => { } else { // Hit the Symon API for receiving Monika report // With the compressed requests and notifications data - await axios({ - data: { + await httpClient(`${url}/api/v1/monika/report`, { + body: JSON.stringify({ data: { notifications, requests, }, monikaId, - }, + }), headers: { 'Content-Type': 'application/json', 'x-api-key': apiKey, }, method: 'POST', - url: `${url}/api/v1/monika/report`, }) log.info( From 38931fa57720fbf578157b41ba240ba5f6b6556d Mon Sep 17 00:00:00 2001 From: Irwansyah Date: Fri, 19 Jul 2024 14:17:33 +0700 Subject: [PATCH 04/10] chore: remove axios in http prober --- src/components/probe/prober/http/request.ts | 242 -------------------- 1 file changed, 242 deletions(-) diff --git a/src/components/probe/prober/http/request.ts b/src/components/probe/prober/http/request.ts index a52fb81f0..22b2d209d 100644 --- a/src/components/probe/prober/http/request.ts +++ b/src/components/probe/prober/http/request.ts @@ -39,7 +39,6 @@ import { icmpRequest } from '../icmp/request' import registerFakes from '../../../../utils/fakes' import { sendHttpRequest } from '../../../../utils/http' import { log } from '../../../../utils/pino' -import { AxiosError } from 'axios' import { getErrorMessage } from '../../../../utils/catch-error-handler' import { HttpClientHeaderList, @@ -121,10 +120,6 @@ export async function httpRequest({ } catch (error: unknown) { const responseTime = Date.now() - startTime - if (error instanceof AxiosError) { - return handleAxiosError(responseTime, error) - } - const { value, error: undiciErrorValidator } = UndiciErrorValidator.validate(error, { allowUnknown: true, @@ -376,243 +371,6 @@ function transformContentByType( } } -function handleAxiosError( - responseTime: number, - error: AxiosError -): ProbeRequestResponse { - // The request was made and the server responded with a status code - // 400, 500 get here - if (error?.response) { - return { - data: '', - body: '', - status: error?.response?.status, - headers: error?.response?.headers, - responseTime, - result: probeRequestResult.success, - error: error?.response?.data as string, - } - } - - // The request was made but no response was received - // timeout is here, ECONNABORTED, ENOTFOUND, ECONNRESET, ECONNREFUSED - if (error?.request) { - const { status, description } = getErrorStatusWithExplanation(error) - return { - data: '', - body: '', - status, - headers: '', - responseTime, - result: probeRequestResult.failed, - error: description, - } - } - - return { - data: '', - body: '', - status: 99, - headers: '', - responseTime, - result: probeRequestResult.failed, - error: getErrorMessage(error), - } -} - -// suppress switch-case complexity since this is dead-simple mapping to error code -// eslint-disable-next-line complexity -function getErrorStatusWithExplanation(error: unknown): { - status: number - description: string -} { - switch ((error as AxiosError).code) { - case 'ECONNABORTED': { - return { - status: 599, - description: - 'ECONNABORTED: The connection was unexpectedly terminated, often due to server issues, network problems, or timeouts. Please check the server status or network connectivity and try again.', - } - } // https://httpstatuses.com/599 - - case 'ENOTFOUND': { - return { - status: 0, - description: - "ENOTFOUND: The monitored website or server couldn't be found, similar to entering an incorrect web address or encountering a temporary network/server issue. Verify the URL and ensure the server is accessible.", - } - } - - case 'ECONNRESET': { - return { - status: 1, - description: - 'ECONNRESET: The connection to a server was unexpectedly reset, often pointing to issues on the server side or network interruptions. Check the server status and network connection, and retry the request.', - } - } - - case 'ECONNREFUSED': { - return { - status: 2, - description: - 'ECONNREFUSED: Attempted to connect to a server, but the server declined the connection. Ensure the server is running and accepting connections.', - } - } - - case 'ERR_FR_TOO_MANY_REDIRECTS': { - return { - status: 3, - description: - 'ERR_FR_TOO_MANY_REDIRECTS: Webpage is stuck in a loop of continuously redirecting. Review the redirection rules and fix any redirect loops.', - } - } - - case 'ERR_BAD_OPTION_VALUE': { - return { - status: 4, - description: - 'ERR_BAD_OPTION_VALUE: Invalid or inappropriate value is provided for an option. Check and correct the option values being passed.', - } - } - - case 'ERR_BAD_OPTION': { - return { - status: 5, - description: - 'ERR_BAD_OPTION: Invalid or inappropriate option is used. Verify the options being used and correct them.', - } - } - - case 'ETIMEDOUT': { - return { - status: 6, - description: - 'ETIMEDOUT: Connection attempt has timed out. Try increasing the timeout value or check the server response time.', - } - } - - case 'ERR_NETWORK': { - return { - status: 7, - description: - 'ERR_NETWORK: Signals a general network-related issue such as poor connectivity, DNS issues, or firewall restrictions. Verify network connectivity and DNS settings, and ensure no firewall is blocking the connection.', - } - } - - case 'ERR_DEPRECATED': { - return { - status: 8, - description: - 'ERR_DEPRECATED: Feature, method, or functionality used in the code is outdated or no longer supported. Update the code to use supported features or methods.', - } - } - - case 'ERR_BAD_RESPONSE': { - return { - status: 9, - description: - 'ERR_BAD_RESPONSE: Server provides a response that cannot be understood or is considered invalid. Ensure the server is providing a valid response.', - } - } - - case 'ERR_BAD_REQUEST': { - return { - status: 11, - description: - "ERR_BAD_REQUEST: Client's request to the server is malformed or invalid. Review and correct the request parameters being sent to the server.", - } - } - - case 'ERR_CANCELED': { - return { - status: 12, - description: - 'ERR_CANCELED: Request or operation is canceled before it completes. Check if the cancellation was intended or adjust the request flow.', - } - } - - case 'ERR_NOT_SUPPORT': { - return { - status: 13, - description: - 'ERR_NOT_SUPPORT: Feature or operation is not supported. Use an alternative feature or operation that is supported.', - } - } - - case 'ERR_INVALID_URL': { - return { - status: 14, - description: - 'ERR_INVALID_URL: URL is not formatted correctly or is not a valid web address. Verify and correct the URL being used.', - } - } - - case 'EAI_AGAIN': { - return { - status: 15, - description: - 'EAI_AGAIN: Temporary failure in resolving a domain name. Retry the request after some time or check DNS settings.', - } - } - - case 'EHOSTUNREACH': { - return { - status: 16, - description: - 'EHOSTUNREACH: The host is unreachable. Verify the network connection and ensure the host is accessible.', - } - } - - case 'EPROTO': { - return { - status: 17, - description: - "EPROTO: There are issues with the website's SSL/TLS certificates, incompatible protocols, or other SSL-related problems. Verify the SSL/TLS certificates and ensure compatible protocols are used.", - } - } - - case 'CERT_HAS_EXPIRED': { - return { - status: 18, - description: - "CERT_HAS_EXPIRED: The website's SSL/TLS certificates has expired. Renew the SSL/TLS certificates to resolve the issue.", - } - } - - case 'UNABLE_TO_VERIFY_LEAF_SIGNATURE': { - return { - status: 27, - description: - 'ELEAFSIGNATURE: Unable to verify the first/leaf certificate. Check the certificate chain and ensure all certificates are valid.', - } - } - - case 'ERR_TLS_CERT_ALTNAME_INVALID': { - return { - status: 28, - description: `ERR_TLS_CERT_ALTNAME_INVALID: Invalid certificate altname. Verify the certificate's subject alternative names and correct any issues.`, - } - } - - default: { - if (error instanceof AxiosError) { - log.error( - `Error code 99: Unhandled error while probing ${error.request.url}, got ${error.code} ${error.stack} ` - ) - } else { - log.error( - `Error code 99: Unhandled error, got ${(error as AxiosError).stack}` - ) - } - - return { - status: 99, - description: `Error code 99: ${(error as AxiosError).stack}`, - } - } // in the event an unlikely unknown error, send here - } -} - function handleUndiciError( responseTime: number, error: undiciErrors.UndiciError From 2dc8b280e05c5398a1ca464f2fcce2de66888a0a Mon Sep 17 00:00:00 2001 From: Irwansyah Date: Fri, 19 Jul 2024 14:17:33 +0700 Subject: [PATCH 05/10] chore: remove axios in utils http wrapper --- src/utils/http.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/utils/http.ts b/src/utils/http.ts index e7ba6b7bf..91e0118c7 100644 --- a/src/utils/http.ts +++ b/src/utils/http.ts @@ -93,6 +93,7 @@ async function fetchRedirect( do { // eslint-disable-next-line no-await-in-loop currentResponse = await fetcher(nextUrl) + // check for HTTP status code 3xx const shouldRedirect = currentResponse.status >= 300 && currentResponse.status < 400 From 9dbe920f39ff3c4363c778f35c56c9728336e449 Mon Sep 17 00:00:00 2001 From: Irwansyah Date: Fri, 19 Jul 2024 14:17:33 +0700 Subject: [PATCH 06/10] fix: return empty string if fetch response is not a valid json --- src/components/http-client/index.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/components/http-client/index.ts b/src/components/http-client/index.ts index d19ec22c5..66905db0e 100644 --- a/src/components/http-client/index.ts +++ b/src/components/http-client/index.ts @@ -150,9 +150,11 @@ export const httpClient = async ( signal: requestOptions.signal, }) - let data + let data: unknown = '' if (!isStreamResponseType) { - data = await fetchResponse.json() + try { + data = await fetchResponse.json() + } catch {} } return new HttpClientResponse(fetchResponse, isStreamResponseType, data) From ac3aca0b9f89485c32a80db6b8f8502789334227 Mon Sep 17 00:00:00 2001 From: Irwansyah Date: Fri, 19 Jul 2024 14:17:33 +0700 Subject: [PATCH 07/10] fix: fetch error because reading body twice --- src/components/http-client/index.ts | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/components/http-client/index.ts b/src/components/http-client/index.ts index 66905db0e..31b092c51 100644 --- a/src/components/http-client/index.ts +++ b/src/components/http-client/index.ts @@ -75,20 +75,13 @@ export class HttpClientResponse { return this._fetchResponse.type } - // get url: string - // readonly redirected: boolean - - // readonly bodyUsed: boolean - - // readonly arrayBuffer: () => Promise - // readonly blob: () => Promise - // readonly formData: () => Promise - json(): Promise { + if (this._data) return Promise.resolve(this._data) return this._fetchResponse.json() } text(): Promise { + if (this._data) return Promise.resolve(this._data as string) return this._fetchResponse.text() } @@ -154,7 +147,9 @@ export const httpClient = async ( if (!isStreamResponseType) { try { data = await fetchResponse.json() - } catch {} + } catch { + data = await fetchResponse.text() + } } return new HttpClientResponse(fetchResponse, isStreamResponseType, data) From fa3477951452eb0c3dc9eec1c556c593e37f414e Mon Sep 17 00:00:00 2001 From: Irwansyah Date: Fri, 19 Jul 2024 14:17:33 +0700 Subject: [PATCH 08/10] fix: retrieving content-type --- src/components/probe/prober/http/request.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/components/probe/prober/http/request.ts b/src/components/probe/prober/http/request.ts index 22b2d209d..6b600f33a 100644 --- a/src/components/probe/prober/http/request.ts +++ b/src/components/probe/prober/http/request.ts @@ -301,9 +301,14 @@ async function probeHttpClient({ } } - const responseBody = ( - (response.headers as HttpClientHeaderList).get('Content-Type') as string - )?.includes('application/json') + let contentType = (response.headers as HttpClientHeaderList).get( + 'Content-Type' + ) + if (!contentType) { + contentType = (response.headers as HttpClientHeaderList).get('content-type') + } + + const responseBody = (contentType as string)?.startsWith('application/json') ? await response.json() : await response.text() From b9f85907e9224b7543e3d20965a8fa7cdf033478 Mon Sep 17 00:00:00 2001 From: Irwansyah Date: Fri, 19 Jul 2024 14:17:33 +0700 Subject: [PATCH 09/10] fix(request.test): pass the followRedirects value --- src/components/probe/prober/http/request.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/components/probe/prober/http/request.test.ts b/src/components/probe/prober/http/request.test.ts index 0f768629f..c9323e200 100644 --- a/src/components/probe/prober/http/request.test.ts +++ b/src/components/probe/prober/http/request.test.ts @@ -480,6 +480,7 @@ describe('probingHTTP', () => { ) const request = { url: 'https://example.com/redirect-1', + followRedirects: 2, } as RequestConfig // act From e4cc7d506a4b7541e5f871dd97407ff46e60bdac Mon Sep 17 00:00:00 2001 From: Irwansyah Date: Thu, 8 Aug 2024 13:50:30 +0700 Subject: [PATCH 10/10] fix: send null body if the requestParams.body is null --- src/components/probe/prober/http/request.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/components/probe/prober/http/request.ts b/src/components/probe/prober/http/request.ts index 6b600f33a..0772f795e 100644 --- a/src/components/probe/prober/http/request.ts +++ b/src/components/probe/prober/http/request.ts @@ -289,7 +289,9 @@ async function probeHttpClient({ body: typeof requestParams.body === 'string' ? requestParams.body - : JSON.stringify(requestParams.body), + : requestParams.body + ? JSON.stringify(requestParams.body) + : null, }) const responseTime = Date.now() - startTime