From 5be8e93fd49574b14bb8fa7d65f4c18baf66b507 Mon Sep 17 00:00:00 2001 From: younglim Date: Thu, 20 Jun 2024 15:10:17 +0800 Subject: [PATCH] Do not continue taking screenshots if element cannot be screenshot --- src/crawlers/custom/utils.ts | 178 ++++++++++++----------- src/screenshotFunc/htmlScreenshotFunc.ts | 15 +- 2 files changed, 106 insertions(+), 87 deletions(-) diff --git a/src/crawlers/custom/utils.ts b/src/crawlers/custom/utils.ts index 09632e47..2b9368ef 100644 --- a/src/crawlers/custom/utils.ts +++ b/src/crawlers/custom/utils.ts @@ -19,67 +19,66 @@ export const log = str => { export const screenshotFullPage = async (page, screenshotsDir:string, screenshotIdx) => { const imgName = `PHScan-screenshot${screenshotIdx}.png`; const imgPath = path.join(screenshotsDir, imgName); - - const fullPageSize = await page.evaluate(() => ({ - width: Math.max( - document.body.scrollWidth, - document.documentElement.scrollWidth, - document.body.offsetWidth, - document.documentElement.offsetWidth, - document.body.clientWidth, - document.documentElement.clientWidth, - ), - height: Math.max( - document.body.scrollHeight, - document.documentElement.scrollHeight, - document.body.offsetHeight, - document.documentElement.offsetHeight, - document.body.clientHeight, - document.documentElement.clientHeight, - ), - })); - const originalSize = page.viewportSize(); - const usesInfiniteScroll = async () => { - const prevHeight = await page.evaluate(() => document.body.scrollHeight); - - await page.evaluate(() => { - window.scrollTo(0, document.body.scrollHeight); - }); + try { + const fullPageSize = await page.evaluate(() => ({ + width: Math.max( + document.body.scrollWidth, + document.documentElement.scrollWidth, + document.body.offsetWidth, + document.documentElement.offsetWidth, + document.body.clientWidth, + document.documentElement.clientWidth, + ), + height: Math.max( + document.body.scrollHeight, + document.documentElement.scrollHeight, + document.body.offsetHeight, + document.documentElement.offsetHeight, + document.body.clientHeight, + document.documentElement.clientHeight, + ), + })); + + const usesInfiniteScroll = async () => { + const prevHeight = await page.evaluate(() => document.body.scrollHeight); + + await page.evaluate(() => { + window.scrollTo(0, document.body.scrollHeight); + }); - const isLoadMoreContent = async () => - new Promise(resolve => { - setTimeout(async () => { - await page.waitForLoadState('domcontentloaded'); + const isLoadMoreContent = async () => + new Promise(resolve => { + setTimeout(async () => { + await page.waitForLoadState('domcontentloaded'); - const newHeight = await page.evaluate( - // eslint-disable-next-line no-shadow - () => document.body.scrollHeight, - ); - const result = newHeight > prevHeight; + const newHeight = await page.evaluate( + // eslint-disable-next-line no-shadow + () => document.body.scrollHeight, + ); + const result = newHeight > prevHeight; - resolve(result); - }, 2500); - }); + resolve(result); + }, 2500); + }); - const result = await isLoadMoreContent(); - return result; - }; + const result = await isLoadMoreContent(); + return result; + }; - await usesInfiniteScroll(); + await usesInfiniteScroll(); - // scroll back to top of page for screenshot - await page.evaluate(() => { - window.scrollTo(0, 0); - }); + // scroll back to top of page for screenshot + await page.evaluate(() => { + window.scrollTo(0, 0); + }); - consoleLogger.info(`Screenshot page at: ${urlWithoutAuth(page.url())}`); - silentLogger.info(`Screenshot page at: ${urlWithoutAuth(page.url())}`); + consoleLogger.info(`Screenshot page at: ${urlWithoutAuth(page.url())}`); + silentLogger.info(`Screenshot page at: ${urlWithoutAuth(page.url())}`); - try { await page.screenshot({ - timeout: 10000, + timeout: 5000, path: imgPath, clip: { x: 0, @@ -95,10 +94,10 @@ export const screenshotFullPage = async (page, screenshotsDir:string, screenshot } catch (e) { consoleLogger.error('Unable to take screenshot'); + // Do not return screenshot path if screenshot fails + return ""; } - if (originalSize) await page.setViewportSize(originalSize); - return `screenshots/${imgName}`; // relative path from reports folder }; @@ -130,7 +129,8 @@ export const runAxeScan = async ( export const processPage = async (page, processPageParams) => { // make sure to update processPageParams' scannedIdx processPageParams.scannedIdx += 1; - const { + + let { scannedIdx, blacklistedPatterns, includeScreenshots, @@ -139,15 +139,14 @@ export const processPage = async (page, processPageParams) => { urlsCrawled, randomToken, } = processPageParams; + try { - await page.waitForLoadState('networkidle', { timeout: 10000 }); - await page.waitForLoadState('domcontentloaded'); + await page.waitForLoadState('domcontentloaded', { timeout: 10000 }); } catch (e) { - consoleLogger.info('Unable to detect networkidle'); - silentLogger.info('Unable to detect networkidle'); + consoleLogger.info('Unable to detect page load state'); } - log(`Scan - processPage: ${page.url()}`); + consoleLogger.info(`Attempting to scan: ${page.url()}`); const pageUrl = page.url(); @@ -170,33 +169,46 @@ export const processPage = async (page, processPageParams) => { // return; // } - const initialScrollPos = await page.evaluate(() => ({ - x: window.scrollX, - y: window.scrollY, - })); + try { + const initialScrollPos = await page.evaluate(() => ({ + x: window.scrollX, + y: window.scrollY, + })); + + const pageImagePath = await screenshotFullPage(page, intermediateScreenshotsPath, scannedIdx); + - const pageImagePath = await screenshotFullPage(page, intermediateScreenshotsPath, scannedIdx); + // TODO: This is a temporary fix to not take element screenshots on pages when errors out at full page screenshot + if (pageImagePath === "") { + includeScreenshots = false; + } - guiInfoLog(guiInfoStatusTypes.SCANNED, { - numScanned: urlsCrawled.scanned.length, - urlScanned: pageUrl, - }); + await runAxeScan( + page, + includeScreenshots, + randomToken, + { + pageIndex: scannedIdx, + pageImagePath, + }, + dataset, + urlsCrawled, + ); - await runAxeScan( - page, - includeScreenshots, - randomToken, - { - pageIndex: scannedIdx, - pageImagePath, - }, - dataset, - urlsCrawled, - ); + guiInfoLog(guiInfoStatusTypes.SCANNED, { + numScanned: urlsCrawled.scanned.length, + urlScanned: pageUrl, + }); + + await page.evaluate(pos => { + window.scrollTo(pos.x, pos.y); + }, initialScrollPos); + + } catch (e) { + consoleLogger.error(`Error in scanning page: ${pageUrl}`); + } + - await page.evaluate(pos => { - window.scrollTo(pos.x, pos.y); - }, initialScrollPos); }; export const MENU_POSITION = { @@ -205,7 +217,6 @@ export const MENU_POSITION = { }; export const updateMenu = async (page, urlsCrawled) => { - await page.waitForLoadState('domcontentloaded'); log(`Overlay menu: updating: ${page.url()}`); await page.evaluate( vars => { @@ -219,7 +230,8 @@ export const updateMenu = async (page, urlsCrawled) => { }, { urlsCrawled }, ); - log(`Overlay menu: updating: success`); + + consoleLogger.info(`Overlay menu updated`); }; export const addOverlayMenu = async (page, urlsCrawled, menuPos) => { diff --git a/src/screenshotFunc/htmlScreenshotFunc.ts b/src/screenshotFunc/htmlScreenshotFunc.ts index 2812722c..320338f1 100644 --- a/src/screenshotFunc/htmlScreenshotFunc.ts +++ b/src/screenshotFunc/htmlScreenshotFunc.ts @@ -1,5 +1,5 @@ // import { JSDOM } from "jsdom"; -import { silentLogger } from '../logs.js'; +import { consoleLogger, silentLogger } from '../logs.js'; import { createHash } from 'crypto'; import fs from 'fs'; import path from 'path'; @@ -22,15 +22,22 @@ export const takeScreenshotForHTMLElements = async (violations, page, randomToke const locator = page.locator(selector); const locators = await locator.all(); for (const currLocator of locators) { - await currLocator.scrollIntoViewIfNeeded({ timeout: locatorTimeout }); + await currLocator.scrollIntoViewIfNeeded({ timeout: locatorTimeout }); + const isVisible = await currLocator.isVisible(); + + if (isVisible) { const buffer = await currLocator.screenshot({ timeout: locatorTimeout }); const screenshotPath = getScreenshotPath(buffer, randomToken); node.screenshotPath = screenshotPath; screenshotCount++; - break; // Stop looping after finding the first visible locator + } else { + consoleLogger.info(`Element at ${currLocator} is not visible`); + } + + break; // Stop looping after finding the first visible locator } } catch (e) { - silentLogger.info(e); + consoleLogger.info(`Unable to take element screenshot at ${selector}`); } } newViolationNodes.push(node);