From 5b38cc6041136c7e0ea3d828863253b7250d6ca8 Mon Sep 17 00:00:00 2001 From: jpreisner Date: Tue, 11 May 2021 16:38:28 +0200 Subject: [PATCH] feat(configurer-proxy): Ajout de la configuration du proxy dans Chromium et dans le Dockerfile --- Dockerfile | 14 ++++++++++++ README.md | 53 +++++++++++++++++++++++++++++++++++++++++++- cli-core/analysis.js | 15 +++++++++---- commands/analyse.js | 31 ++++++++++++++++++++------ greenit | 5 +++++ 5 files changed, 106 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 422d825..0c6f3f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,13 @@ # See : https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#running-puppeteer-in-docker FROM node:14-slim + +# Uncomment if you need to configure proxy. +# You can init these variables by using --build-args during docker build +# Example : docker build [...] --build-args http_proxy=http://:@: +#ENV HTTP_PROXY=$http_proxy +#ENV HTTPS_PROXY=$https_proxy +#ENV NO_PROXY=$no_proxy + RUN apt-get update \ && apt-get install -y wget gnupg \ && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ @@ -10,13 +18,19 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* WORKDIR /app COPY . . + +# Uncomment if you need to configure proxy. +#RUN npm config set proxy $HTTP_PROXY + RUN npm i \ && npm link \ && groupadd -r pptruser && useradd -r -g pptruser -G audio,video pptruser \ && mkdir -p /home/pptruser/Downloads \ && chown -R pptruser:pptruser /home/pptruser \ && chown -R pptruser:pptruser /app/ + USER pptruser + # To avoid "Error: ENOENT: no such file or directory, open '/app/dist/bundle.js'" RUN npm i diff --git a/README.md b/README.md index b05f0bf..f8ec306 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ Cette application est basée sur l'extension Chrome GreenIT-Analysis (https://gi - [Docker](#docker) - [Prérequis](#prérequis-1) - [Utilisation](#utilisation) + - [Configurer un proxy](#configurer-un-proxy) - [Usage](#usage) - [Analyse](#analyse) - [Prérequis](#prérequis-2) @@ -84,6 +85,37 @@ npm link ``` docker build -t imageName . ``` + +### Configurer un proxy +Si vous avez besoin de configurer un proxy, il faut : + +1. Modifier le Dockerfile + +``` +# Uncomment if you need to configure proxy. +# You can init these variables by using --build-arg during docker build +# Example : docker build [...] --build-arg http_proxy=http://:@: +ENV HTTP_PROXY=$http_proxy +ENV HTTPS_PROXY=$https_proxy +ENV NO_PROXY=$no_proxy + +[...] + +# Uncomment if you need to configure proxy. +#RUN npm config set proxy $HTTP_PROXY +``` + +2. Construire l'image en passant les informations du proxy en paramètres + +Exemple : +``` +docker build -t imageName \ + --build-arg http_proxy=http://:@: \ + --build-arg https_proxy=https://:@: \ + --build-arg no_proxy= \ + . +``` + # Usage ## Analyse @@ -158,6 +190,15 @@ Paramètres optionnels : - iPhoneX - iPad +- `--proxy , -p` : Chemin vers le fichier YAML contenant les informations de configuration du proxy. + + Exemple de proxy.yaml : + ```yaml + server: ":" + user: "" + password: "" + ``` + ### Usage avec Docker 1. Déposer le fichier `` dans le dossier `//input`. 2. Lancer l'analyse : @@ -171,7 +212,6 @@ docker run -it --init --rm --cap-add=SYS_ADMIN \ 3. Récupérer les résultats dans votre dossier `//output` #### Redéfinir les variables `URL_PATH` et `RESULTS_PATH` - Vous pouvez redéfinir les variables `URL_PATH` et `RESULTS_PATH` si vous souhaitez changer le nom des fichiers ou leur emplacement. Exemple : @@ -198,6 +238,17 @@ docker run -it --init --rm --cap-add=SYS_ADMIN \ greenit analyse /app/input/url.yaml /app/output/results.xlsx --max_tab=1 --timeout=15000 --retry=5 ``` +#### Lancer l'analyse avec la configuration d'un proxy +Vous pouvez déposer le fichier `proxy.yaml` dans le dossier `//input` et lancer le conteneur : +``` +docker run -it --init --rm --cap-add=SYS_ADMIN \ + -v //input:/app/input \ + -v //output:/app/output \ + --name containerName \ + imageName \ + greenit analyse /app/input/url.yaml /app/output/results.xlsx --proxy=/app/input/proxy.yaml +``` + ## ParseSiteMap ``` diff --git a/cli-core/analysis.js b/cli-core/analysis.js index fceaef4..d0a0a7d 100644 --- a/cli-core/analysis.js +++ b/cli-core/analysis.js @@ -16,9 +16,13 @@ async function analyseURL(browser, pageInformations, options) { const TAB_ID = options.tabId const TRY_NB = options.tryNb || 1 const DEVICE = options.device || "desktop" + const PROXY = options.proxy try { const page = await browser.newPage(); + if (PROXY) { + await page.authenticate({ username: PROXY.user, password: PROXY.password }); + } await page.setViewport(sizes[DEVICE]); // disabling cache @@ -112,7 +116,7 @@ async function login(browser,loginInformations) { } //Core -async function createJsonReports(browser, pagesInformations, options) { +async function createJsonReports(browser, pagesInformations, options, proxy) { //Timeout for an analysis const TIMEOUT = options.timeout; //Concurent tab @@ -159,7 +163,8 @@ async function createJsonReports(browser, pagesInformations, options) { asyncFunctions.push(analyseURL(browser,pagesInformations[index],{ device: DEVICE, timeout:TIMEOUT, - tabId: i + tabId: i, + proxy: proxy })); index++; //console.log(`Start of analysis #${index}/${pagesInformations.length}`) @@ -172,7 +177,8 @@ async function createJsonReports(browser, pagesInformations, options) { device: DEVICE, timeout:TIMEOUT, tabId: results.tabId, - tryNb: results.tryNb + 1 + tryNb: results.tryNb + 1, + proxy: proxy })); // convert is NEEDED, variable size array }else{ let filePath = path.resolve(SUBRESULTS_DIRECTORY,`${resultId}.json`) @@ -194,7 +200,8 @@ async function createJsonReports(browser, pagesInformations, options) { asyncFunctions.splice(results.tabId,1,analyseURL(browser,pagesInformations[index],{ device: DEVICE, timeout:TIMEOUT, - tabId: results.tabId + tabId: results.tabId, + proxy: proxy })); // No need for convert, fixed size array index++; //console.log(`Start of analysis #${index}/${pagesInformations.length}`) diff --git a/commands/analyse.js b/commands/analyse.js index 3c5faf5..86eef65 100644 --- a/commands/analyse.js +++ b/commands/analyse.js @@ -17,15 +17,32 @@ async function analyse_core(options) { throw ` yaml_input_file : "${URL_YAML_FILE}" is not a valid YAML file.` } + let browserArgs = [ + "--no-sandbox", // can't run inside docker without + "--disable-setuid-sandbox" // but security issues + ] + + // Add proxy conf in browserArgs + let proxy = {}; + if(options.proxy) { + const PROXY_FILE = path.resolve(options.proxy); + try { + proxy = YAML.parse(fs.readFileSync(PROXY_FILE).toString()); + if (!proxy.server || !proxy.user || !proxy.password) { + throw `proxy_config_file : Bad format "${PROXY_FILE}". Expected server, user and password.` + } + browserArgs.push(`--proxy-server=${proxy.server}`); + } catch (error) { + throw ` proxy_config_file : "${PROXY_FILE}" is not a valid YAML file.` + } + } + //start browser const browser = await puppeteer.launch({ - headless:true, - args :[ - "--no-sandbox", // can't run inside docker without - "--disable-setuid-sandbox" // but security issues - ], + headless: true, + args: browserArgs, // Keep gpu horsepower in headless - ignoreDefaultArgs:[ + ignoreDefaultArgs: [ '--disable-gpu' ] }); @@ -45,7 +62,7 @@ async function analyse_core(options) { await login(browser, loginInfos) } //analyse - reports = await createJsonReports(browser, pagesInformations, options); + reports = await createJsonReports(browser, pagesInformations, options, proxy); } finally { //close browser let pages = await browser.pages(); diff --git a/greenit b/greenit index fef1533..a749394 100644 --- a/greenit +++ b/greenit @@ -53,6 +53,11 @@ yargs(hideBin(process.argv)) choices:Object.keys(sizes), default: "desktop" }) + .option('proxy', { + alias: 'p', + type: 'string', + description: 'Path to YAML file with proxy configuration to apply in Chromium' + }) }, (argv) => { require("./commands/analyse.js")(argv) })