Skip to content

Commit

Permalink
feat(trace-viewer): render iframe canvas in trace viewer (#33809)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ruifigueira
ruifigueira authored Dec 13, 2024
1 parent 65688d6 commit c700a84
Show file tree
Hide file tree
Showing 4 changed files with 87 additions and 24 deletions.
12 changes: 6 additions & 6 deletions packages/playwright-core/src/server/trace/recorder/snapshotterInjected.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -438,13 +438,13 @@ export function frameSnapshotStreamer(snapshotStreamer: string, removeNoScript:
expectValue(value);
attrs[kSelectedAttribute] = value;
}
if (nodeName === 'CANVAS') {
const boundingRect = (element as HTMLCanvasElement).getBoundingClientRect();
if (nodeName === 'CANVAS' || nodeName === 'IFRAME' || nodeName === 'FRAME') {
const boundingRect = (element as HTMLElement).getBoundingClientRect();
const value = JSON.stringify({
left: boundingRect.left / window.innerWidth,
top: boundingRect.top / window.innerHeight,
right: boundingRect.right / window.innerWidth,
bottom: boundingRect.bottom / window.innerHeight
left: boundingRect.left,
top: boundingRect.top,
right: boundingRect.right,
bottom: boundingRect.bottom
});
expectValue(kBoundingRectAttribute);
expectValue(value);
Expand Down
87 changes: 72 additions & 15 deletions packages/trace-viewer/src/sw/snapshotRenderer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ export class SnapshotRenderer {
const html = prefix + [
// Hide the document in order to prevent flickering. We will unhide once script has processed shadow.
'<style>*,*::before,*::after { visibility: hidden }</style>',
`<script>${snapshotScript(this._callId, this.snapshotName)}</script>`
`<script>${snapshotScript(this.viewport(), this._callId, this.snapshotName)}</script>`
].join('') + result.join('');
return { value: html, size: html.length };
});
Expand Down Expand Up @@ -236,10 +236,39 @@ function snapshotNodes(snapshot: FrameSnapshot): NodeSnapshot[] {
return (snapshot as any)._nodes;
}

function snapshotScript(...targetIds: (string | undefined)[]) {
function applyPlaywrightAttributes(unwrapPopoutUrl: (url: string) => string, ...targetIds: (string | undefined)[]) {
type ViewportSize = { width: number, height: number };
type BoundingRect = { left: number, top: number, right: number, bottom: number };
type FrameBoundingRectsInfo = {
viewport: ViewportSize;
frames: WeakMap<Element, {
boundingRect: BoundingRect;
scrollLeft: number;
scrollTop: number;
}>;
};

declare global {
interface Window {
__playwright_frame_bounding_rects__: FrameBoundingRectsInfo;
}
}

function snapshotScript(viewport: ViewportSize, ...targetIds: (string | undefined)[]) {
function applyPlaywrightAttributes(unwrapPopoutUrl: (url: string) => string, viewport: ViewportSize, ...targetIds: (string | undefined)[]) {
const isUnderTest = new URLSearchParams(location.search).has('isUnderTest');

// info to recursively compute canvas position relative to the top snapshot frame.
// Before rendering each iframe, its parent extracts the '__playwright_canvas_render_info__' attribute
// value and keeps in this variable. It can then remove the attribute and render the element,
// which will eventually trigger the same process inside the iframe recursively.
// When there's a canvas to render, we iterate over its ancestor frames to compute
// its position relative to the top snapshot frame.
const frameBoundingRectsInfo = {
viewport,
frames: new WeakMap(),
};
window['__playwright_frame_bounding_rects__'] = frameBoundingRectsInfo;

const kPointerWarningTitle = 'Recorded click position in absolute coordinates did not' +
' match the center of the clicked element. This is likely due to a difference between' +
' the test runner and the trace viewer operating systems.';
Expand All @@ -249,6 +278,10 @@ function snapshotScript(...targetIds: (string | undefined)[]) {
const targetElements: Element[] = [];
const canvasElements: HTMLCanvasElement[] = [];

let topSnapshotWindow: Window = window;
while (topSnapshotWindow !== topSnapshotWindow.parent && !topSnapshotWindow.location.pathname.match(/\/page@[a-z0-9]+$/))
topSnapshotWindow = topSnapshotWindow.parent;

const visit = (root: Document | ShadowRoot) => {
// Collect all scrolled elements for later use.
for (const e of root.querySelectorAll(`[__playwright_scroll_top_]`))
Expand Down Expand Up @@ -288,6 +321,11 @@ function snapshotScript(...targetIds: (string | undefined)[]) {
}

for (const iframe of root.querySelectorAll('iframe, frame')) {
const boundingRectJson = iframe.getAttribute('__playwright_bounding_rect__');
iframe.removeAttribute('__playwright_bounding_rect__');
const boundingRect = boundingRectJson ? JSON.parse(boundingRectJson) : undefined;
if (boundingRect)
frameBoundingRectsInfo.frames.set(iframe, { boundingRect, scrollLeft: 0, scrollTop: 0 });
const src = iframe.getAttribute('__playwright_src__');
if (!src) {
iframe.setAttribute('src', 'data:text/html,<body style="background: #ddd"></body>');
Expand Down Expand Up @@ -339,16 +377,20 @@ function snapshotScript(...targetIds: (string | undefined)[]) {
for (const element of scrollTops) {
element.scrollTop = +element.getAttribute('__playwright_scroll_top_')!;
element.removeAttribute('__playwright_scroll_top_');
if (frameBoundingRectsInfo.frames.has(element))
frameBoundingRectsInfo.frames.get(element)!.scrollTop = element.scrollTop;
}
for (const element of scrollLefts) {
element.scrollLeft = +element.getAttribute('__playwright_scroll_left_')!;
element.removeAttribute('__playwright_scroll_left_');
if (frameBoundingRectsInfo.frames.has(element))
frameBoundingRectsInfo.frames.get(element)!.scrollLeft = element.scrollTop;
}

document.styleSheets[0].disabled = true;

const search = new URL(window.location.href).searchParams;
const isTopFrame = window.location.pathname.match(/\/page@[a-z0-9]+$/);
const isTopFrame = window === topSnapshotWindow;

if (search.get('pointX') && search.get('pointY')) {
const pointX = +search.get('pointX')!;
Expand Down Expand Up @@ -419,16 +461,6 @@ function snapshotScript(...targetIds: (string | undefined)[]) {
context.fillRect(0, 0, canvas.width, canvas.height);
}


if (!isTopFrame) {
for (const canvas of canvasElements) {
const context = canvas.getContext('2d')!;
drawCheckerboard(context, canvas);
canvas.title = `Playwright displays canvas contents on a best-effort basis. It doesn't support canvas elements inside an iframe yet. If this impacts your workflow, please open an issue so we can prioritize.`;
}
return;
}

const img = new Image();
img.onload = () => {
for (const canvas of canvasElements) {
Expand All @@ -446,6 +478,31 @@ function snapshotScript(...targetIds: (string | undefined)[]) {
continue;
}

let currWindow: Window = window;
while (currWindow !== topSnapshotWindow) {
const iframe = currWindow.frameElement!;
currWindow = currWindow.parent;

const iframeInfo = currWindow['__playwright_frame_bounding_rects__']?.frames.get(iframe);
if (!iframeInfo?.boundingRect)
break;

const leftOffset = iframeInfo.boundingRect.left - iframeInfo.scrollLeft;
const topOffset = iframeInfo.boundingRect.top - iframeInfo.scrollTop;

boundingRect.left += leftOffset;
boundingRect.top += topOffset;
boundingRect.right += leftOffset;
boundingRect.bottom += topOffset;
}

const { width, height } = topSnapshotWindow['__playwright_frame_bounding_rects__'].viewport;

boundingRect.left = boundingRect.left / width;
boundingRect.top = boundingRect.top / height;
boundingRect.right = boundingRect.right / width;
boundingRect.bottom = boundingRect.bottom / height;

const partiallyUncaptured = boundingRect.right > 1 || boundingRect.bottom > 1;
const fullyUncaptured = boundingRect.left > 1 || boundingRect.top > 1;
if (fullyUncaptured) {
Expand Down Expand Up @@ -483,7 +540,7 @@ function snapshotScript(...targetIds: (string | undefined)[]) {
window.addEventListener('DOMContentLoaded', onDOMContentLoaded);
}

return `\n(${applyPlaywrightAttributes.toString()})(${unwrapPopoutUrl.toString()}${targetIds.map(id => `, "${id}"`).join('')})`;
return `\n(${applyPlaywrightAttributes.toString()})(${unwrapPopoutUrl.toString()}, ${JSON.stringify(viewport)}${targetIds.map(id => `, "${id}"`).join('')})`;
}


Expand Down
6 changes: 4 additions & 2 deletions tests/library/snapshotter.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -268,10 +268,12 @@ it.describe('snapshots', () => {
});
});

function distillSnapshot(snapshot, distillTarget = true) {
function distillSnapshot(snapshot, options: { distillTarget: boolean, distillBoundingRect: boolean } = { distillTarget: true, distillBoundingRect: true }) {
let { html } = snapshot.render();
if (distillTarget)
if (options.distillTarget)
html = html.replace(/\s__playwright_target__="[^"]+"/g, '');
if (options.distillBoundingRect)
html = html.replace(/\s__playwright_bounding_rect__="[^"]+"/g, '');
return html
.replace(/<style>\*,\*::before,\*::after { visibility: hidden }<\/style>/, '')
.replace(/<script>[.\s\S]+<\/script>/, '')
Expand Down
6 changes: 5 additions & 1 deletion tests/library/trace-viewer.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1539,12 +1539,16 @@ test('canvas clipping in iframe', async ({ runAndTrace, page, server }) => {
await page.setContent(`
<iframe src="${server.PREFIX}/screenshots/canvas.html#canvas-on-edge"></iframe>
`);
await page.locator('iframe').contentFrame().locator('canvas').scrollIntoViewIfNeeded();
await rafraf(page, 5);
});

const msg = await traceViewer.page.waitForEvent('console', { predicate: msg => msg.text().startsWith('canvas drawn:') });
expect(msg.text()).toEqual('canvas drawn: [1,1,11,20]');

const snapshot = await traceViewer.snapshotFrame('page.evaluate');
const canvas = snapshot.locator('iframe').contentFrame().locator('canvas');
await expect(canvas).toHaveAttribute('title', `Playwright displays canvas contents on a best-effort basis. It doesn't support canvas elements inside an iframe yet. If this impacts your workflow, please open an issue so we can prioritize.`);
await expect(canvas).toHaveAttribute('title', 'Canvas contents are displayed on a best-effort basis based on viewport screenshots taken during test execution.');
});

test('should show only one pointer with multilevel iframes', async ({ page, runAndTrace, server, browserName }) => {
Expand Down

0 comments on commit c700a84

Please sign in to comment.