diff --git a/.env b/.env new file mode 100644 index 0000000..8802390 --- /dev/null +++ b/.env @@ -0,0 +1,23 @@ +# InfluxDB +# Ne pas modifier les deux variables suivante +INFLUXDB_HOST=greenit-cli-influxdb +INFLUXDB_PORT=8086 + +INFLUXDB_ORG_NAME=default +INFLUXDB_USERNAME=default +INFLUXDB_PASSWORD=defaultinfluxdb +INFLUXDB_BUCKET_NAME=db0 +INFLUXDB_RETENTION=24w + +# Renseigner ces variables une fois influxdb démarré pour la première fois +INFLUXDB_TOKEN=token +INFLUXDB_ORG_ID=orgId + +# Grafana +GRAFANA_USERNAME=admin +GRAFANA_PASSWORD=admin + +# Image Version +# Attention lors des changements de versions +INFLUXDB_IMAGE_VERSION=influxdb:2.1.1 +GRAFANA_IMAGE_VERSION=grafana/grafana:8.4.3 \ No newline at end of file diff --git a/.gitignore b/.gitignore index c7d7086..6d7ea39 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ dist/ results/ results_test/ *.xlsx -*.yaml \ No newline at end of file +*.yaml +!docker-compose.yaml \ No newline at end of file diff --git a/README.md b/README.md index bcb3552..fd1508c 100644 --- a/README.md +++ b/README.md @@ -328,6 +328,7 @@ Paramètres optionnels : Choix : - xlsx - html +- influxdb - `--headers , -h` : Chemin vers le fichier YAML contenant les headers HTTP configurés pour accéder aux URL à analyser. @@ -338,6 +339,12 @@ Choix : accept-language: 'en-US,en;q=0.9,en;q=0.8' ``` +- `--influxdb` : Active l'écriture des données dans une base influxdb +- `--influxdb_hostname` : URL de la base influxdb +- `--influxdb_org` : Nom de l'organisation influxdb +- `--influxdb_token` : Token de connexion pour influxdb +- `--influxdb_bucket` : Bucket infludb sur lequel envoyer les données + ### Usage avec Docker 1. Déposer le fichier `` dans le dossier `//input`. 2. Lancer l'analyse : @@ -450,6 +457,27 @@ Exemple d'un rapport : ![Page d'une URL analysée dans le rapport HTML](./docs/rapport-html-detail-page.png) +#### InfluxDB +Prérequis : +- Le paramètre suivant est définit : `--format=influxdb` ou `-f=influxdb` + +Les données seront envoyés sur influxdb. + +Un docker-compose avec un exemple de configuration d'influxdb et de grafana est présent dans le projet. Lors de la première utilisation, quelques étapes de mise en place sont nécessaires : +- Changer les couples nom d'utilisateur/mot de passe dans le fichier .env (optionel) ; +- Démarrer le conteneur influxdb : `docker-compose up greenit-cli-influxdb` ; +- Se connecter à influxdb (`http://localhost:8086` par défault) pour récupérer l'id de l'organisation (dans l'url après la connexion `http://localhost:8086/orgs/`) et le token de connection (data -> API Token), et renseigner les variables d'environnement correspondantes ; +- Il est ensuite possible de démarrer le conteneur grafana et d'envoyer les données sur influxdb. + +Ces étapes ne seront pas nécessaire à nouveau. Il faudra toutefois redémarrer au moins le conteneur influxdb avant un test. + +Exemple d'usage : +```shell +greenit analyse exampleUrl.yaml --format=influxdb --influxdb_hostname http://localhost:8086 --influxdb_org organisation --influxdb_token token --influxdb_bucket db0 +``` +Exemple de dashboard grafana (l'url testée est celle du site d'[ecoindex](http://ecoindex.fr/)) +![Page d'une URL analysée dans le rapport HTML](./docs/grafana-dashboard.png) + ## ParseSiteMap ``` diff --git a/cli-core/influxdb.js b/cli-core/influxdb.js new file mode 100644 index 0000000..43d9e6e --- /dev/null +++ b/cli-core/influxdb.js @@ -0,0 +1,91 @@ +const {InfluxDB, Point, HttpError} = require('@influxdata/influxdb-client'); +const fs = require('fs'); +const ProgressBar = require('progress'); + +async function write(reports, options) { + + if (!options.influxdb_hostname) { + throw `You must define an InfluxDB hostname.`; + } + if (!options.influxdb_token) { + throw `You must define an InfluxDB token.`; + } + if (!options.influxdb_bucket) { + throw `You must define an InfluxDB bucket name.`; + } + if (!options.influxdb_org) { + throw `You must define an InfluxDB organisation.`; + } + + const url = options.influxdb_hostname; + + //initialise progress bar + let progressBar; + if (!options.ci){ + progressBar = new ProgressBar(' Push to InfluxDB [:bar] :percent Remaining: :etas Time: :elapseds', { + complete: '=', + incomplete: ' ', + width: 40, + total: reports.length+2 + }); + progressBar.tick(); + } else { + console.log('Push report to InfluxDB ...'); + } + + // initialise client + const client = new InfluxDB({url: options.influxdb_hostname, token: options.influxdb_token}); + const writeApi = client.getWriteApi(options.influxdb_org, options.influxdb_bucket); + + // create points from reports + const points = reports.map((file) => { + let obj = JSON.parse(fs.readFileSync(file.path).toString()); + let hostname = obj.url.split('/')[2]; + let point = new Point("eco_index") + .tag("pageName", obj.pageInformations.name) + .tag("hostname", hostname) + .stringField("url", obj.url) + .stringField("hostname", hostname) + .stringField("grade", obj.grade) + .intField("ecoindex", obj.ecoIndex) + .floatField("water", obj.waterConsumption) + .floatField("ges", obj.greenhouseGasesEmission) + .floatField("domSize", obj.domSize) + .stringField("pageSize", `${Math.round(obj.responsesSize / 1000)} (${Math.round(obj.responsesSizeUncompress / 1000)})`) + .floatField("nbRequest", obj.nbRequest) + .floatField("nbPlugins", obj.pluginsNumber) + .floatField("cssFilesNumber", obj.printStyleSheetsNumber) + .floatField("cssInlineNumber", obj.inlineStyleSheetsNumber) + .floatField("emptySrcTagNumber", obj.emptySrcTagNumber) + .floatField("inlineJsScriptsNumber", obj.inlineJsScriptsNumber) + .floatField("responsesSize", Math.round(obj.responsesSize / 1000)) + .floatField("responsesSizeUncompress", Math.round(obj.responsesSizeUncompress / 1000)); + + Object.keys(obj.bestPractices) + .map(key => point.stringField(key, obj.bestPractices[key].complianceLevel || 'A')); + + if (progressBar) progressBar.tick() + + return point + }); + + //upload points and close connexion + writeApi.writePoints(points); + + writeApi + .close() + .then(() => { + if (progressBar) progressBar.tick(); + }) + .catch(e => { + console.log('Writing to influx failed\n'); + console.error(e); + if (e instanceof HttpError && e.statusCode === 401) { + console.log(`The InfluxDB database: ${bucket} doesn't exist.`); + } + }); +} + +module.exports = { + write +} \ No newline at end of file diff --git a/commands/analyse.js b/commands/analyse.js index 5952355..69a98a1 100644 --- a/commands/analyse.js +++ b/commands/analyse.js @@ -7,6 +7,7 @@ const login = require('../cli-core/analysis.js').login; const create_global_report = require('../cli-core/reportGlobal.js').create_global_report; const create_XLSX_report = require('../cli-core/reportExcel.js').create_XLSX_report; const create_html_report = require('../cli-core/reportHtml.js').create_html_report; +const writeToInflux = require("../cli-core/influxdb").write; //launch core async function analyse_core(options) { @@ -76,9 +77,13 @@ async function analyse_core(options) { let reportObj = await create_global_report(reports, options); if (reportFormat === 'html') { await create_html_report(reportObj, options); - } else { + } else if (reportFormat === 'influxdb') { + await writeToInflux(reports, options); + } + else { await create_XLSX_report(reportObj, options); } + } function readProxy(proxyFile) { @@ -109,7 +114,7 @@ function readHeaders(headersFile) { function getReportFormat(format, filename) { // Check if format is defined - const formats = ['xlsx', 'html']; + const formats = ['xlsx', 'html', 'influxdb']; if (format && formats.includes(format.toLowerCase())) { return format.toLowerCase(); } diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..b5413e2 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,44 @@ +services: + greenit-cli-influxdb: + image: ${INFLUXDB_IMAGE_VERSION} + container_name: ${INFLUXDB_HOST} + ports: + - '8086:8086' + volumes: + - greenit-cli-influxdb-storage:/var/lib/influxdb2 + - ./influxdb/queries:/home/queries:rw + environment: + - DOCKER_INFLUXDB_INIT_MODE=setup + - DOCKER_INFLUXDB_INIT_USERNAME=${INFLUXDB_USERNAME} + - DOCKER_INFLUXDB_INIT_PASSWORD=${INFLUXDB_PASSWORD} + - DOCKER_INFLUXDB_INIT_ORG=${INFLUXDB_ORG_NAME} + - DOCKER_INFLUXDB_INIT_BUCKET=${INFLUXDB_BUCKET_NAME} + - DOCKER_INFLUXDB_INIT_RETENTION=${INFLUXDB_RETENTION} + + greenit-cli-grafana: + container_name: greenit-cli-grafana + image: ${GRAFANA_IMAGE_VERSION} + ports: + - '3000:3000' + volumes: + - greenit-cli-grafana-storage:/var/lib/grafana + - ./grafana-provisioning/:/etc/grafana/provisioning + depends_on: + - greenit-cli-influxdb + environment: + - GF_SECURITY_ADMIN_USER=${GRAFANA_USERNAME} + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD} + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_USERS_ALLOW_SIGN_UP=false + - GF_USERS_ALLOW_ORG_CREATE=false + - INFLUXDB_ORG_ID=${INFLUXDB_ORG_ID} + - INFLUXDB_TOKEN=${INFLUXDB_TOKEN} + - INFLUXDB_BUCKET_NAME=${INFLUXDB_BUCKET_NAME} + - INFLUXDB_HOST=${INFLUXDB_HOST} + - INFLUXDB_PORT=${INFLUXDB_PORT} + +volumes: + greenit-cli-influxdb-storage: + driver: local + greenit-cli-grafana-storage: + driver: local \ No newline at end of file diff --git a/docs/grafana-dashboard.png b/docs/grafana-dashboard.png new file mode 100644 index 0000000..67d57dc Binary files /dev/null and b/docs/grafana-dashboard.png differ diff --git a/grafana-provisioning/dashboards/dashboard.yml b/grafana-provisioning/dashboards/dashboard.yml new file mode 100644 index 0000000..045a5f2 --- /dev/null +++ b/grafana-provisioning/dashboards/dashboard.yml @@ -0,0 +1,10 @@ +apiVersion: 1 +providers: +- name: InfluxDB + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /etc/grafana/provisioning/dashboards + diff --git a/grafana-provisioning/dashboards/ecoindex_dashboard.json b/grafana-provisioning/dashboards/ecoindex_dashboard.json new file mode 100644 index 0000000..cf3e9d0 --- /dev/null +++ b/grafana-provisioning/dashboards/ecoindex_dashboard.json @@ -0,0 +1,1472 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "iteration": 1649665107917, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "Eco Index", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "blue", + "value": 70 + }, + { + "color": "purple", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.4.3", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "eco_index", + "orderByTime": "ASC", + "policy": "default", + "query": "from(bucket: v.defaultBucket)\n |> range(start: $__from/1000, stop: $__to/1000)\n |> filter(fn: (r) => r._measurement == \"eco_index\" and r.pageName == \"${pagename}\" and r._field == \"ecoindex\")\n |> last()", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "EcoIndex" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 25, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^Grade$/", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.3", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "from(bucket: v.defaultBucket)\n |> range(start: $__from/1000, stop: $__to/1000)\n |> filter(fn: (r) => r._measurement == \"eco_index\" and r.pageName == \"${pagename}\" and r._field == \"grade\")\n |> last()", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "grade .*", + "renamePattern": "Grade" + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "GES (gCO2e)", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(128, 133, 128)", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.4.3", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "from(bucket: v.defaultBucket)\n |> range(start: $__from/1000, stop: $__to/1000)\n |> filter(fn: (r) => r._measurement == \"eco_index\" and r.pageName == \"${pagename}\" and r._field == \"ges\")\n |> last()\n", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "Water (cl)", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.4.3", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "from(bucket: v.defaultBucket)\n |> range(start: $__from/1000, stop: $__to/1000)\n |> filter(fn: (r) => r._measurement == \"eco_index\" and r.pageName == \"${pagename}\" and r._field == \"water\")\n |> last()", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": "" + } + ] + }, + "unit": "string" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "_field" + }, + "properties": [ + { + "id": "custom.width", + "value": 267 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "_value" + }, + "properties": [ + { + "id": "custom.width", + "value": 265 + } + ] + } + ] + }, + "gridPos": { + "h": 22, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 30, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 1, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "responsesSizeUncompress {_start=\"2022-03-10 11:33:45.165130575 +0000 UTC\", _stop=\"2022-03-17 11:33:45.165130575 +0000 UTC\"}" + } + ] + }, + "pluginVersion": "8.4.3", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "eco_index", + "orderByTime": "ASC", + "policy": "default", + "query": "from(bucket: v.defaultBucket)\n |> range(start: $__from/1000, stop: $__to/1000)\n |> filter(fn: (r) => r._measurement == \"eco_index\" and r.pageName == \"${pagename}\" and r._field =~ /^[A-Z]\\w*$/ )\n |> last()\n |> group()\n |> drop(columns: [\"_start\", \"_stop\", \"_time\", \"_measurement\", \"hostname\", \"pageName\"])", + "rawQuery": true, + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "UseStandardTypefaces" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "title": "Best Practices", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "pageName": true + }, + "indexByName": { + "_field": 0, + "_value": 1 + }, + "renameByName": { + "_field": "Domaine", + "_value": "Note" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "DOM Size", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 6 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.4.3", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "from(bucket: v.defaultBucket)\n |> range(start: $__from/1000, stop: $__to/1000)\n |> filter(fn: (r) => r._measurement == \"eco_index\" and r.pageName == \"${pagename}\" and r._field == \"domSize\")\n |> last()", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "Responses Size (kb)", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 6 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.4.3", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "from(bucket: v.defaultBucket)\n |> range(start: $__from/1000, stop: $__to/1000)\n |> filter(fn: (r) => r._measurement == \"eco_index\" and r.pageName == \"${pagename}\" and r._field == \"responsesSize\")\n |> last()", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "Response Size Uncompress (kb)", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 8, + "y": 6 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.4.3", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "from(bucket: v.defaultBucket)\n |> range(start: $__from/1000, stop: $__to/1000)\n |> filter(fn: (r) => r._measurement == \"eco_index\" and r.pageName == \"${pagename}\" and r._field == \"responsesSizeUncompress\")\n |> last()", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "Number of request", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 6 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.4.3", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "from(bucket: v.defaultBucket)\n |> range(start: $__from/1000, stop: $__to/1000)\n |> filter(fn: (r) => r._measurement == \"eco_index\" and r.pageName == \"${pagename}\" and r._field == \"nbRequest\")\n |> last()", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "Plugins Number", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 0, + "y": 12 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.4.3", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "from(bucket: v.defaultBucket)\n |> range(start: $__from/1000, stop: $__to/1000)\n |> filter(fn: (r) => r._measurement == \"eco_index\" and r.pageName == \"${pagename}\" and r._field == \"nbPlugins\")\n |> last()", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "JS Scripts Inline Number", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 3, + "y": 12 + }, + "id": 28, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.4.3", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "from(bucket: v.defaultBucket)\n |> range(start: $__from/1000, stop: $__to/1000)\n |> filter(fn: (r) => r._measurement == \"eco_index\" and r.pageName == \"${pagename}\" and r._field == \"inlineJsScriptsNumber\")\n |> last()", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "CSS Inline Number", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 7, + "y": 12 + }, + "id": 27, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.4.3", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "from(bucket: v.defaultBucket)\n |> range(start: $__from/1000, stop: $__to/1000)\n |> filter(fn: (r) => r._measurement == \"eco_index\" and r.pageName == \"${pagename}\" and r._field == \"cssInlineNumber\")\n |> last()", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "CSS Files Number", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 10, + "y": 12 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.4.3", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "from(bucket: v.defaultBucket)\n |> range(start: $__from/1000, stop: $__to/1000)\n |> filter(fn: (r) => r._measurement == \"eco_index\" and r.pageName == \"${pagename}\" and r._field == \"cssFilesNumber\")\n |> last()", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "Empty Src Tag Number", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 13, + "y": 12 + }, + "id": 29, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.4.3", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "from(bucket: v.defaultBucket)\n |> range(start: $__from/1000, stop: $__to/1000)\n |> filter(fn: (r) => r._measurement == \"eco_index\" and r.pageName == \"${pagename}\" and r._field == \"emptySrcTagNumber\")\n |> last()", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "type": "stat" + } + ], + "refresh": false, + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "Liollury", + "value": "Liollury" + }, + "datasource": { + "type": "influxdb", + "uid": "PC003803D09A91473" + }, + "definition": "import \"influxdata/influxdb/schema\"\n\nschema.tagValues(\n bucket: v.defaultBucket,\n tag: \"pageName\",\n start: -1y\n)\n", + "hide": 0, + "includeAll": false, + "label": "Nom de la page", + "multi": false, + "name": "pagename", + "options": [], + "query": "import \"influxdata/influxdb/schema\"\n\nschema.tagValues(\n bucket: v.defaultBucket,\n tag: \"pageName\",\n start: -1y\n)\n", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "EcoIndex", + "uid": "kLmqUTyGk", + "version": 3, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana-provisioning/datasources/datasource.yml b/grafana-provisioning/datasources/datasource.yml new file mode 100644 index 0000000..29cb862 --- /dev/null +++ b/grafana-provisioning/datasources/datasource.yml @@ -0,0 +1,13 @@ +apiVersion: 1 +datasources: + - name: InfluxDB2 + type: influxdb + access: proxy + url: http://${INFLUXDB_HOST}:${INFLUXDB_PORT} + secureJsonData: + token: ${INFLUXDB_TOKEN} + jsonData: + version: Flux + organization: ${INFLUXDB_ORG_ID} + defaultBucket: ${INFLUXDB_BUCKET_NAME} + editable: true diff --git a/greenit b/greenit index 111275e..4782c54 100644 --- a/greenit +++ b/greenit @@ -61,13 +61,25 @@ yargs(hideBin(process.argv)) .option('format', { alias: 'f', type: 'string', - description: 'Report format : Possible choices : xlsx (excel), html' + description: 'Report format : Possible choices : xlsx (excel), html, influxdb' }) .option('headers', { alias: 'h', type: 'string', description: 'Headers HTTP to configure to analyze url', }) + .option('influxdb_hostname', { + type: 'string' + }) + .option('influxdb_bucket', { + type: 'string' + }) + .option('influxdb_token', { + type: 'string' + }) + .option('influxdb_org', { + type: 'string' + }) }, (argv) => { require("./commands/analyse.js")(argv) }) diff --git a/package.json b/package.json index b4b0810..3e25ffe 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,8 @@ "author": "", "license": "ISC", "dependencies": { + "@influxdata/influxdb-client": "^1.20.0", + "@influxdata/influxdb-client-apis": "^1.20.0", "axios": "^0.21.1", "concat-files": "^0.1.1", "exceljs": "^4.2.0",