Merge pull request cnumr#5 from jpreisner/feat/configurer-proxy

Feat/configurer proxy
Palo-IT · Jun 9, 2021 · dfde7e1 · dfde7e1
2 parents 020595a + 5b38cc6
commit dfde7e1
Show file tree

Hide file tree

Showing 5 changed files with 106 additions and 12 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,13 @@
 # See : https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#running-puppeteer-in-docker
 FROM node:14-slim
+
+# Uncomment if you need to configure proxy. 
+# You can init these variables by using --build-args during docker build
+# Example : docker build [...] --build-args http_proxy=http://<user>:<password>@<host>:<port>
+#ENV HTTP_PROXY=$http_proxy
+#ENV HTTPS_PROXY=$https_proxy
+#ENV NO_PROXY=$no_proxy
+
 RUN apt-get update \
     && apt-get install -y wget gnupg \
     && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
@@ -10,13 +18,19 @@ RUN apt-get update \
     && rm -rf /var/lib/apt/lists/*
 WORKDIR /app
 COPY . .
+
+# Uncomment if you need to configure proxy. 
+#RUN npm config set proxy $HTTP_PROXY 
+
 RUN npm i \
     && npm link \
     && groupadd -r pptruser && useradd -r -g pptruser -G audio,video pptruser \
     && mkdir -p /home/pptruser/Downloads \
     && chown -R pptruser:pptruser /home/pptruser \
     && chown -R pptruser:pptruser /app/
+
 USER pptruser
+
 # To avoid "Error: ENOENT: no such file or directory, open '/app/dist/bundle.js'"
 RUN npm i
 

diff --git a/README.md b/README.md
@@ -10,6 +10,7 @@ Cette application est basée sur l'extension Chrome GreenIT-Analysis (https://gi
   - [Docker](#docker)
     - [Prérequis](#prérequis-1)
     - [Utilisation](#utilisation)
+    - [Configurer un proxy](#configurer-un-proxy)
 - [Usage](#usage)
   - [Analyse](#analyse)
     - [Prérequis](#prérequis-2)
@@ -84,6 +85,37 @@ npm link
  ```
  docker build -t imageName .
  ```
+
+### Configurer un proxy
+Si vous avez besoin de configurer un proxy, il faut :
+
+1. Modifier le Dockerfile 
+
+```
+# Uncomment if you need to configure proxy. 
+# You can init these variables by using --build-arg during docker build
+# Example : docker build [...] --build-arg http_proxy=http://<user>:<password>@<host>:<port>
+ENV HTTP_PROXY=$http_proxy
+ENV HTTPS_PROXY=$https_proxy
+ENV NO_PROXY=$no_proxy
+
+[...]
+
+# Uncomment if you need to configure proxy. 
+#RUN npm config set proxy $HTTP_PROXY 
+```
+
+2. Construire l'image en passant les informations du proxy en paramètres
+
+Exemple :
+```
+docker build -t imageName \
+  --build-arg http_proxy=http://<user>:<password>@<host>:<port> \
+  --build-arg https_proxy=https://<user>:<password>@<host>:<port> \
+  --build-arg no_proxy=<no_proxy> \
+  .
+```
+
 # Usage
 
 ## Analyse
@@ -158,6 +190,15 @@ Paramètres optionnels :
   - iPhoneX
   - iPad
 
+- `--proxy , -p` : Chemin vers le fichier YAML contenant les informations de configuration du proxy.
+
+  Exemple de proxy.yaml :
+  ```yaml
+  server: "<host>:<port>"
+  user: "<username>"
+  password: "<password>"
+  ```
+
 ### Usage avec Docker
 1. Déposer le fichier `<yaml_input_file>` dans le dossier `/<path>/input`.
 2. Lancer l'analyse :
@@ -171,7 +212,6 @@ docker run -it --init --rm --cap-add=SYS_ADMIN \
 3. Récupérer les résultats dans votre dossier `/<path>/output`
 
 #### Redéfinir les variables `URL_PATH` et `RESULTS_PATH` 
-
 Vous pouvez redéfinir les variables `URL_PATH` et `RESULTS_PATH` si vous souhaitez changer le nom des fichiers ou leur emplacement.
 
 Exemple :
@@ -198,6 +238,17 @@ docker run -it --init --rm --cap-add=SYS_ADMIN \
   greenit analyse /app/input/url.yaml /app/output/results.xlsx --max_tab=1 --timeout=15000 --retry=5
 ```
 
+#### Lancer l'analyse avec la configuration d'un proxy
+Vous pouvez déposer le fichier `proxy.yaml` dans le dossier `/<path>/input` et lancer le conteneur :
+```
+docker run -it --init --rm --cap-add=SYS_ADMIN \
+  -v /<path>/input:/app/input \
+  -v /<path>/output:/app/output  \
+  --name containerName \
+  imageName \
+  greenit analyse /app/input/url.yaml /app/output/results.xlsx --proxy=/app/input/proxy.yaml
+```
+
 ## ParseSiteMap
 
 ```

diff --git a/cli-core/analysis.js b/cli-core/analysis.js
@@ -16,9 +16,13 @@ async function analyseURL(browser, pageInformations, options) {
     const TAB_ID = options.tabId
     const TRY_NB =  options.tryNb || 1
     const DEVICE = options.device || "desktop"
+    const PROXY = options.proxy
 
     try {
         const page = await browser.newPage();
+        if (PROXY) {
+            await page.authenticate({ username: PROXY.user, password: PROXY.password });
+        }
         await page.setViewport(sizes[DEVICE]);
 
         // disabling cache
@@ -112,7 +116,7 @@ async function login(browser,loginInformations) {
 }
 
 //Core
-async function createJsonReports(browser, pagesInformations, options) {
+async function createJsonReports(browser, pagesInformations, options, proxy) {
     //Timeout for an analysis
     const TIMEOUT = options.timeout;
     //Concurent tab
@@ -159,7 +163,8 @@ async function createJsonReports(browser, pagesInformations, options) {
         asyncFunctions.push(analyseURL(browser,pagesInformations[index],{
             device: DEVICE,
             timeout:TIMEOUT,
-            tabId: i
+            tabId: i,
+            proxy: proxy
         }));
         index++;
         //console.log(`Start of analysis #${index}/${pagesInformations.length}`)
@@ -172,7 +177,8 @@ async function createJsonReports(browser, pagesInformations, options) {
                 device: DEVICE,
                 timeout:TIMEOUT,
                 tabId: results.tabId,
-                tryNb: results.tryNb + 1
+                tryNb: results.tryNb + 1,
+                proxy: proxy
             })); // convert is NEEDED, variable size array
         }else{
             let filePath = path.resolve(SUBRESULTS_DIRECTORY,`${resultId}.json`)
@@ -194,7 +200,8 @@ async function createJsonReports(browser, pagesInformations, options) {
                 asyncFunctions.splice(results.tabId,1,analyseURL(browser,pagesInformations[index],{
                     device: DEVICE,
                     timeout:TIMEOUT,
-                    tabId: results.tabId
+                    tabId: results.tabId,
+                    proxy: proxy
                 })); // No need for convert, fixed size array
                 index++;
                 //console.log(`Start of analysis #${index}/${pagesInformations.length}`)

diff --git a/commands/analyse.js b/commands/analyse.js
@@ -17,15 +17,32 @@ async function analyse_core(options) {
         throw ` yaml_input_file : "${URL_YAML_FILE}" is not a valid YAML file.`
     }
 
+    let browserArgs = [
+        "--no-sandbox",                 // can't run inside docker without
+        "--disable-setuid-sandbox"      // but security issues
+    ]
+
+    // Add proxy conf in browserArgs
+    let proxy = {};
+    if(options.proxy) {
+        const PROXY_FILE = path.resolve(options.proxy);
+        try {
+            proxy = YAML.parse(fs.readFileSync(PROXY_FILE).toString());
+            if (!proxy.server || !proxy.user || !proxy.password) {
+                throw `proxy_config_file : Bad format "${PROXY_FILE}". Expected server, user and password.`
+            }
+            browserArgs.push(`--proxy-server=${proxy.server}`);
+        } catch (error) {
+            throw ` proxy_config_file : "${PROXY_FILE}" is not a valid YAML file.`
+        }
+    }
+
     //start browser
     const browser = await puppeteer.launch({
-        headless:true,
-        args :[
-            "--no-sandbox",                 // can't run inside docker without
-            "--disable-setuid-sandbox"      // but security issues
-        ],
+        headless: true,
+        args: browserArgs,
         // Keep gpu horsepower in headless
-        ignoreDefaultArgs:[
+        ignoreDefaultArgs: [
             '--disable-gpu'
         ]
     });
@@ -45,7 +62,7 @@ async function analyse_core(options) {
             await login(browser, loginInfos)
         }
         //analyse
-        reports = await createJsonReports(browser, pagesInformations, options);
+        reports = await createJsonReports(browser, pagesInformations, options, proxy);
     } finally {
         //close browser
         let pages = await browser.pages();

diff --git a/greenit b/greenit
@@ -53,6 +53,11 @@ yargs(hideBin(process.argv))
         choices:Object.keys(sizes),
         default: "desktop"
       })
+      .option('proxy', {
+        alias: 'p',
+        type: 'string',
+        description: 'Path to YAML file with proxy configuration to apply in Chromium'
+      })
   }, (argv) => {
       require("./commands/analyse.js")(argv)
   })