feat(http prober): #1285 implement cache http response (#1296)

docs/src/pages/guides/cli-options.md

@@ -336,6 +336,17 @@ If there is a probe with request(s) that uses HTTPS, Monika will show an error i
 monika --ignoreInvalidTLS
 ```
 
+## TTL Cache
+
+Time-to-live for in-memory (HTTP) cache entries in minutes. Defaults to 5 minutes. Setting to 0 means disabling this cache. This cache is used for requests with identical HTTP request config, e.g. headers, method, url.
+
+Only usable for probes which does not have [chaining requests.](https://hyperjumptech.github.io/monika/guides/examples#requests-chaining)
+
+```bash
+# Set TTL cache for HTTP to 5 minutes
+monika --ttl-cache 5
+```
+
 ## Verbose
 
 Like your app to be more chatty and honest revealing all its internal details? Use the `--verbose` flag.
@@ -344,6 +355,16 @@ Like your app to be more chatty and honest revealing all its internal details? U
 monika --verbose
 ```
 
+## Verbose Cache
+
+Show (HTTP) cache hit / miss messages to log.
+
+This will only show for probes which does not have [chaining requests.](https://hyperjumptech.github.io/monika/guides/examples#requests-chaining)
+
+```bash
+monika --verbose-cache
+```
+
 ## Version
 
 The `-v` or `--version` flag prints the current application version.

docs/src/pages/guides/probes.md

@@ -75,6 +75,12 @@ Details of the field are given in the table below.
 | allowUnauthorized (optional) | (boolean), If set to true, will make https agent to not check for ssl certificate validity                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
 | followRedirects (optional)   | The request follows redirects as many times as specified here. If unspecified, it will fallback to the value set by the [follow redirects flag](https://monika.hyperjump.tech/guides/cli-options#follow-redirects)                                                                                                                                                                                                                                                                                                                                                                                |
 
+### Good to know
+
+To reduce network usage, HTTP responses are cached with 5 time-to-live by default. This cache is then reused for requests with identical HTTP request config, e.g. headers, method, url.
+
+This cache is usable for probes which does not have [chaining requests.](https://hyperjumptech.github.io/monika/guides/examples#requests-chaining)
+
 ## Request Body
 
 By default, the request body will be treated as-is. If the request header's `Content-Type` is set to `application/x-www-form-urlencoded`, it will be serialized into URL-safe string in UTF-8 encoding. Body payloads will vary on the specific probes being requested. For HTTP requests, the body and headers are defined like this:

package.json

@@ -59,6 +59,7 @@
   "dependencies": {
     "@faker-js/faker": "^7.4.0",
     "@hyperjumptech/monika-notification": "^1.18.0",
+    "@isaacs/ttlcache": "^1.4.1",
     "@oclif/core": "3.16.0",
     "@oclif/plugin-help": "^6.0.9",
     "@oclif/plugin-version": "^2.0.11",

src/components/probe/prober/http/index.ts

@@ -40,6 +40,7 @@ import { addIncident } from '../../../incident'
 import { saveProbeRequestLog } from '../../../logger/history'
 import { logResponseTime } from '../../../logger/response-time-log'
 import { httpRequest } from './request'
+import { getCache, putCache } from './response-cache'
 
 type ProbeResultMessageParams = {
   request: RequestConfig
@@ -52,14 +53,26 @@ export class HTTPProber extends BaseProber {
     // sending multiple http requests for request chaining
     const responses: ProbeRequestResponse[] = []
 
-    for (const requestConfig of requests) {
-      responses.push(
-        // eslint-disable-next-line no-await-in-loop
-        await httpRequest({
-          requestConfig: { ...requestConfig, signal },
-          responses,
-        })
-      )
+    // do http request
+    // force fresh request if :
+    // - probe has chaining requests, OR
+    // - this is a retrying attempt
+    if (requests.length > 1 || incidentRetryAttempt > 0) {
+      for (const requestConfig of requests) {
+        responses.push(
+          // eslint-disable-next-line no-await-in-loop
+          await this.doRequest(requestConfig, signal, responses)
+        )
+      }
+    }
+    // use cached response when possible
+    // or fallback to fresh request if cache expired
+    else {
+      const responseCache = getCache(requests[0])
+      const response =
+        responseCache || (await this.doRequest(requests[0], signal, responses))
+      if (!responseCache) putCache(requests[0], response)
+      responses.push(response)
     }
 
     const hasFailedRequest = responses.find(
@@ -165,6 +178,17 @@ export class HTTPProber extends BaseProber {
     }
   }
 
+  private doRequest(
+    config: RequestConfig,
+    signal: AbortSignal | undefined,
+    responses: ProbeRequestResponse[]
+  ) {
+    return httpRequest({
+      requestConfig: { ...config, signal },
+      responses,
+    })
+  }
+
   generateVerboseStartupMessage(): string {
     const { description, id, interval, name } = this.probeConfig
 

src/components/probe/prober/http/request.ts

@@ -103,23 +103,23 @@ export async function httpRequest({
     }
 
     // Do the request using compiled URL and compiled headers (if exists)
-    if (getContext().flags['native-fetch']) {
-      return await probeHttpFetch({
-        startTime,
-        maxRedirects: followRedirects,
-        renderedURL,
-        requestParams: { ...newReq, headers: requestHeaders },
-        allowUnauthorized,
-      })
-    }
-
-    return await probeHttpAxios({
-      startTime,
-      maxRedirects: followRedirects,
-      renderedURL,
-      requestParams: { ...newReq, headers: requestHeaders },
-      allowUnauthorized,
-    })
+    const response = await (getContext().flags['native-fetch']
+      ? probeHttpFetch({
+          startTime,
+          maxRedirects: followRedirects,
+          renderedURL,
+          requestParams: { ...newReq, headers: requestHeaders },
+          allowUnauthorized,
+        })
+      : probeHttpAxios({
+          startTime,
+          maxRedirects: followRedirects,
+          renderedURL,
+          requestParams: { ...newReq, headers: requestHeaders },
+          allowUnauthorized,
+        }))
+
+    return response
   } catch (error: unknown) {
     const responseTime = Date.now() - startTime
 
@@ -372,7 +372,7 @@ function transformContentByType(
     case 'multipart/form-data': {
       const form = new FormData()
       for (const contentKey of Object.keys(content)) {
-        form.append(contentKey, (content as any)[contentKey])
+        form.append(contentKey, (content as never)[contentKey])
       }
 
       return { content: form, contentType: form.getHeaders()['content-type'] }

src/components/probe/prober/http/response-cache.ts

@@ -0,0 +1,69 @@
+/**********************************************************************************
+ * MIT License                                                                    *
+ *                                                                                *
+ * Copyright (c) 2021 Hyperjump Technology                                        *
+ *                                                                                *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy   *
+ * of this software and associated documentation files (the "Software"), to deal  *
+ * in the Software without restriction, including without limitation the rights   *
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell      *
+ * copies of the Software, and to permit persons to whom the Software is          *
+ * furnished to do so, subject to the following conditions:                       *
+ *                                                                                *
+ * The above copyright notice and this permission notice shall be included in all *
+ * copies or substantial portions of the Software.                                *
+ *                                                                                *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR     *
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,       *
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE    *
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER         *
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,  *
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE  *
+ * SOFTWARE.                                                                      *
+ **********************************************************************************/
+
+import { getContext } from '../../../../context'
+import { ProbeRequestResponse, RequestConfig } from 'src/interfaces/request'
+import { log } from '../../../../utils/pino'
+import TTLCache from '@isaacs/ttlcache'
+import { createHash } from 'crypto'
+
+const ttlCache = new TTLCache()
+const cacheHash = new Map<RequestConfig, string>()
+
+function getOrCreateHash(config: RequestConfig) {
+  let hash = cacheHash.get(config)
+  if (!hash) {
+    hash = createHash('SHA1').update(JSON.stringify(config)).digest('hex')
+  }
+
+  return hash
+}
+
+function put(config: RequestConfig, value: ProbeRequestResponse) {
+  if (!getContext().flags['ttl-cache'] || getContext().isTest) return
+  const hash = getOrCreateHash(config)
+  // manually set time-to-live for each cache entry
+  // moved from "new TTLCache()" initialization above because corresponding flag is not yet parsed
+  const ttl = getContext().flags['ttl-cache'] * 60_000
+  ttlCache.set(hash, value, { ttl })
+}
+
+function get(config: RequestConfig): ProbeRequestResponse | undefined {
+  if (!getContext().flags['ttl-cache'] || getContext().isTest) return undefined
+  const key = getOrCreateHash(config)
+  const response = ttlCache.get(key)
+  const isVerbose = getContext().flags['verbose-cache']
+  const shortHash = key.slice(Math.max(0, key.length - 7))
+  if (isVerbose && response) {
+    const time = new Date().toISOString()
+    log.info(`${time} - [${shortHash}] Cache HIT`)
+  } else if (isVerbose) {
+    const time = new Date().toISOString()
+    log.info(`${time} - [${shortHash}] Cache MISS`)
+  }
+
+  return response as ProbeRequestResponse | undefined
+}
+
+export { put as putCache, get as getCache }

src/flag.ts

@@ -44,6 +44,7 @@ export type MonikaFlags = {
   force: boolean
   har?: string
   id?: string
+  ignoreInvalidTLS: boolean
   insomnia?: string
   'keep-verbose-logs': boolean
   logs: boolean
@@ -67,8 +68,9 @@ export type MonikaFlags = {
   symonGetProbesIntervalMs: number
   symonUrl?: string
   text?: string
-  ignoreInvalidTLS: boolean
+  'ttl-cache': number
   verbose: boolean
+  'verbose-cache': boolean
 }
 
 const DEFAULT_CONFIG_INTERVAL_SECONDS = 900
@@ -98,7 +100,9 @@ export const monikaFlagsDefaultValue: MonikaFlags = {
   symonGetProbesIntervalMs: 60_000,
   symonReportInterval: DEFAULT_SYMON_REPORT_INTERVAL_MS,
   symonReportLimit: 100,
+  'ttl-cache': 5,
   verbose: false,
+  'verbose-cache': false,
 }
 
 function getDefaultConfig(): Array<string> {
@@ -276,10 +280,18 @@ export const flags = {
     description: 'Run Monika using a Simple text file',
     exclusive: ['postman', 'insomnia', 'sitemap', 'har'],
   }),
+  'ttl-cache': Flags.integer({
+    description: `Time-to-live for in-memory (HTTP) cache entries in minutes. Defaults to ${monikaFlagsDefaultValue['ttl-cache']} minutes`,
+    default: monikaFlagsDefaultValue['ttl-cache'],
+  }),
   verbose: Flags.boolean({
     default: monikaFlagsDefaultValue.verbose,
     description: 'Show verbose log messages',
   }),
+  'verbose-cache': Flags.boolean({
+    default: monikaFlagsDefaultValue.verbose,
+    description: 'Show cache hit / miss messages to log',
+  }),
   version: Flags.version({ char: 'v' }),
 }