Dev: Fix flaky and broken E2E tests (#9183)

changelog/dev-fix-klarna-and-refund-failures-e2e-tests

@@ -0,0 +1,4 @@
+Significance: patch
+Type: dev
+
+Fix Klarna and Refund Failures E2E tests

tests/e2e-pw/utils/shopper.ts

@@ -99,6 +99,10 @@ export const confirmCardAuthentication = async (
 ): Promise< void > => {
 	// Stripe card input also uses __privateStripeFrame as a prefix, so need to make sure we wait for an iframe that
 	// appears at the top of the DOM.
+	await page.waitForSelector(
+		'body > div > iframe[name^="__privateStripeFrame"]'
+	);
+
 	const stripeFrame = page.frameLocator(
 		'body>div>iframe[name^="__privateStripeFrame"]'
 	);
@@ -112,5 +116,9 @@ export const confirmCardAuthentication = async (
 	const button = challengeFrame.getByRole( 'button', {
 		name: authorize ? 'Complete' : 'Fail',
 	} );
+
+	// Not ideal, but we need to wait for the loading animation to finish before we can click the button.
+	await page.waitForTimeout( 1000 );
+
 	await button.click();
 };

tests/e2e/config/jest.setup.js

@@ -36,6 +36,7 @@ const ERROR_MESSAGES_TO_IGNORE = [
 	'Failed to load resource: the server responded with a status of 400 (Bad Request)',
 	'No Amplitude API key provided',
 	'is registered with an invalid category',
+	'"Heading" is not a supported class',
 ];
 
 ERROR_MESSAGES_TO_IGNORE.forEach( ( errorMessage ) => {
@@ -114,7 +115,8 @@ function removePageEvents() {
 function setTestTimeouts() {
 	const TIMEOUT = 100000;
 	// Increase default value to avoid test failing due to timeouts.
-	page.setDefaultTimeout( TIMEOUT );
+	// But we want the matchers timeout to be smaller than the test timeout to have meaningful error messages.
+	page.setDefaultTimeout( TIMEOUT / 4 );
 	// running the login flow takes more than the default timeout of 5 seconds,
 	// so we need to increase it to run the login in the beforeAll hook
 	jest.setTimeout( TIMEOUT );

tests/e2e/specs/wcpay/merchant/merchant-orders-refund-failures.spec.js

@@ -35,7 +35,6 @@ const dataTable = [
 
 describe( 'Order > Refund Failure', () => {
 	beforeAll( async () => {
-		page.setDefaultTimeout( 10000 );
 		// Place an order to refund later
 		await setupProductCheckout(
 			config.get( 'addresses.customer.billing' )
@@ -65,7 +64,6 @@ describe( 'Order > Refund Failure', () => {
 			}
 		} );
 		await merchant.logout();
-		page.setDefaultTimeout( 100000 );
 	} );
 
 	describe.each( dataTable )(
@@ -78,8 +76,28 @@ describe( 'Order > Refund Failure', () => {
 				// We need to remove any listeners on the `dialog` event otherwise we can't catch the dialog below
 				await page.removeAllListeners( 'dialog' );
 
+				// Sometimes the element is not clickable due to the header getting on the way. This seems to
+				// only happen in CI for WC 7.7.0 so the workaround is to remove those elements.
+				const hideElementIfExists = ( sel ) => {
+					const element = document.querySelector( sel );
+					if ( element ) {
+						element.outerHTML = '';
+					}
+				};
+				await page.evaluate(
+					hideElementIfExists,
+					'.woocommerce-layout__header'
+				);
+				await page.evaluate( hideElementIfExists, '#wpadminbar' );
+
 				// Click the Refund button
-				await expect( page ).toClick( 'button.refund-items' );
+				const refundItemsButton = await expect( page ).toMatchElement(
+					'button.refund-items',
+					{
+						visible: true,
+					}
+				);
+				await refundItemsButton.click();
 
 				// Verify the refund section shows
 				await page.waitForSelector( 'div.wc-order-refund-items' );
@@ -92,25 +110,25 @@ describe( 'Order > Refund Failure', () => {
 				// Initiate refund attempt
 				await expect( page ).toFill( selector, value );
 
-				await expect( page ).toMatchElement( '.do-api-refund', {
-					text: /Refund .* via WooPayments/,
-				} );
+				const refundButton = await expect( page ).toMatchElement(
+					'.do-api-refund',
+					{
+						visible: true,
+						text: /Refund .* via WooPayments/,
+					}
+				);
 
 				// Confirm the refund
 				const refundDialog = await expect( page ).toDisplayDialog(
 					async () => {
-						await expect( page ).toClick( 'button.do-api-refund' );
+						await refundButton.click();
 					}
 				);
 
 				// Confirm that the "Invalid refund amount" alert is shown, then close it
 				const invalidRefundAlert = await expect( page ).toDisplayDialog(
 					async () => {
 						await refundDialog.accept();
-						// await uiUnblocked();
-						// await page.waitForNavigation( {
-						// 	waitUntil: 'networkidle0',
-						// } );
 					}
 				);
 				await expect( invalidRefundAlert.message() ).toEqual(

tests/e2e/specs/wcpay/merchant/merchant-payment-gateways-confirmation.spec.js

@@ -13,6 +13,33 @@ const WC_GATEWAYS_LIST_TABLE__WC_PAYMENTS_TOGGLE =
 	'tr[data-gateway_id="woocommerce_payments"] .wc-payment-gateway-method-toggle-enabled';
 
 describe( 'payment gateways disable confirmation', () => {
+	// Newer WooCommerce versions get rid of the 'Save Changes' button and save the changes immediately
+	const saveChangesIfAvailable = async () => {
+		const saveChangesSelector =
+			"xpath/.//button[contains(., 'Save changes')]";
+		const saveChangesButton = await page.$( saveChangesSelector );
+
+		if ( saveChangesButton ) {
+			const isDisabled = await page.$eval(
+				saveChangesSelector,
+				( node ) => node.disabled
+			);
+
+			if ( ! isDisabled ) {
+				return await Promise.all( [
+					expect( page ).toClick( 'button', {
+						text: 'Save changes',
+					} ),
+					page.waitForNavigation( {
+						waitUntil: 'networkidle0',
+					} ),
+				] );
+			}
+		}
+
+		await page.reload( { waitUntil: 'networkidle0' } );
+	};
+
 	beforeAll( async () => {
 		await merchant.login();
 	} );
@@ -50,12 +77,9 @@ describe( 'payment gateways disable confirmation', () => {
 
 		// After clicking "Cancel", the modal should close and WCPay should still be enabled, even after refresh
 		await expect( page ).not.toMatchTextContent( 'Disable WooPayments' );
-		await expect( page ).toClick( 'button', {
-			text: 'Save changes',
-		} );
-		await page.waitForNavigation( {
-			waitUntil: 'networkidle0',
-		} );
+
+		await saveChangesIfAvailable();
+
 		await expect(
 			page
 		).toMatchElement(
@@ -92,13 +116,8 @@ describe( 'payment gateways disable confirmation', () => {
 			`${ WC_GATEWAYS_LIST_TABLE__WC_PAYMENTS_TOGGLE } .woocommerce-input-toggle:not(.woocommerce-input-toggle--loading)`
 		);
 
-		// and refreshing the page should show WCPay become disabled
-		await expect( page ).toClick( 'button', {
-			text: 'Save changes',
-		} );
-		await page.waitForNavigation( {
-			waitUntil: 'networkidle0',
-		} );
+		await saveChangesIfAvailable();
+
 		await expect(
 			page
 		).toMatchElement(
@@ -113,12 +132,7 @@ describe( 'payment gateways disable confirmation', () => {
 		await page.waitForSelector(
 			`${ WC_GATEWAYS_LIST_TABLE__WC_PAYMENTS_TOGGLE } .woocommerce-input-toggle:not(.woocommerce-input-toggle--loading)`
 		);
-		await expect( page ).toClick( 'button', {
-			text: 'Save changes',
-		} );
-		await page.waitForNavigation( {
-			waitUntil: 'networkidle0',
-		} );
+		await saveChangesIfAvailable();
 		await expect(
 			page
 		).toMatchElement(

tests/e2e/specs/wcpay/merchant/merchant-progressive-onboarding.spec.js

@@ -37,10 +37,14 @@ describe( 'Admin merchant progressive onboarding', () => {
 		// pick Individual business entity
 		await expect( page ).toClick( '[name="business_type"]' );
 		await page.waitForSelector(
-			'[name="business_type"] ~ ul li.components-custom-select-control__item'
+			'[name="business_type"] ~ ul li.components-custom-select-control__item',
+			{ text: /Individual/ }
 		);
-		await expect( page ).toClick(
-			'[name="business_type"] ~ ul li.components-custom-select-control__item'
+		await expect(
+			page
+		).toClick(
+			'[name="business_type"] ~ ul li.components-custom-select-control__item',
+			{ text: /Individual/ }
 		);
 		// pick Software type of goods (MCC)
 		await expect( page ).toClick( '[name="mcc"]' );

tests/e2e/specs/wcpay/shopper/shopper-klarna-checkout.spec.js

@@ -112,32 +112,49 @@ describe( 'Klarna checkout', () => {
 			.waitForSelector( '#onContinue' )
 			.then( ( button ) => button.click() );
 
-		await page.waitForTimeout( 2000 );
-
 		// This is where the OTP code is entered.
 		await page.waitForSelector( '#phoneOtp' );
 
-		await page.waitForTimeout( 2000 );
+		await page.waitForTimeout( 2000 ); // Wait for animations
 
 		await expect( page ).toFill( 'input#otp_field', '123456' );
 
-		// Select Payment Plan - 4 weeks & click continue.
-		await page
-			.waitForSelector( '[id="payinparts_kp.0-ui"] input[type="radio"]' )
-			.then( ( radio ) => radio.click() );
+		await page.waitForSelector( '[role="heading"]', {
+			visible: true,
+			text: /(Confirm and pay)|(Choose how to pay)/i,
+		} );
 
-		await page.waitForTimeout( 2000 );
+		let readyToBuy;
+		try {
+			await page.waitForSelector( 'button#buy_button', {
+				visible: true,
+				timeout: 10000,
+			} );
+			readyToBuy = true;
+		} catch ( err ) {
+			console.warn( err );
+			readyToBuy = false;
+		}
 
-		await page
-			.waitForSelector( 'button[data-testid="pick-plan"]' )
-			.then( ( button ) => button.click() );
+		if ( ! readyToBuy ) {
+			// Select Payment Plan - 4 weeks & click continue.
+			await page
+				.waitForSelector(
+					'[id="payinparts_kp.0-ui"] input[type="radio"]'
+				)
+				.then( ( radio ) => radio.click() );
 
-		await page.waitForTimeout( 2000 );
+			await page.waitForTimeout( 2000 );
+
+			await expect( page ).toClick( 'button', {
+				text: 'Continue',
+			} );
+		}
 
 		// Confirm payment.
-		await page
-			.waitForSelector( 'button#buy_button' )
-			.then( ( button ) => button.click() );
+		await page.waitForSelector( 'button#buy_button' );
+		await page.waitForTimeout( 2000 ); // We need to wait a bit for the button to become clickable.
+		await expect( page ).toClick( 'button#buy_button' );
 
 		// Wait for the order confirmation page to load.
 		await page.waitForNavigation( {

tests/e2e/specs/wcpay/shopper/shopper-multi-currency-widget.spec.js

@@ -97,25 +97,31 @@ describe( 'Shopper Multi-Currency widget', () => {
 					visible: true,
 					timeout: 5000,
 				} );
-				await page.select( '.widget select[name=currency]', 'EUR' );
-				await page.waitForNavigation( { waitUntil: 'networkidle0' } );
+				await Promise.all( [
+					page.select( '.widget select[name=currency]', 'EUR' ),
+					page.waitForNavigation( { waitUntil: 'networkidle0' } ),
+				] );
 				await expect( page.url() ).toContain(
 					`${ url }/?currency=EUR`
 				);
 				await page.waitForSelector(
 					'.widget select[name=currency] option[value=EUR][selected]'
 				);
 				// Change it back to USD for the other tests.
-				await page.select( '.widget select[name=currency]', 'USD' );
-				await page.waitForNavigation( { waitUntil: 'networkidle0' } );
+				await Promise.all( [
+					page.select( '.widget select[name=currency]', 'USD' ),
+					page.waitForNavigation( { waitUntil: 'networkidle0' } ),
+				] );
 			} );
 		}
 	);
 
 	it( 'should not affect prices when currency switching on My account > Orders', async () => {
 		await shopper.login();
-		await page.select( '.widget select[name=currency]', 'USD' );
-		await page.waitForNavigation( { waitUntil: 'networkidle0' } );
+		await Promise.all( [
+			page.select( '.widget select[name=currency]', 'USD' ),
+			page.waitForNavigation( { waitUntil: 'networkidle0' } ),
+		] );
 		await setupProductCheckout(
 			config.get( 'addresses.customer.billing' )
 		);
@@ -135,8 +141,10 @@ describe( 'Shopper Multi-Currency widget', () => {
 		);
 
 		await shopperWCP.goToOrders();
-		await page.select( '.widget select[name=currency]', 'EUR' );
-		await page.waitForNavigation( { waitUntil: 'networkidle0' } );
+		await Promise.all( [
+			page.select( '.widget select[name=currency]', 'EUR' ),
+			page.waitForNavigation( { waitUntil: 'networkidle0' } ),
+		] );
 		await page.waitForSelector(
 			'.widget select[name=currency] option[value=EUR][selected]'
 		);
@@ -166,8 +174,10 @@ describe( 'Shopper Multi-Currency widget', () => {
 			)
 		);
 
-		await page.select( '.widget select[name=currency]', 'EUR' );
-		await page.waitForNavigation( { waitUntil: 'networkidle0' } );
+		await Promise.all( [
+			page.select( '.widget select[name=currency]', 'EUR' ),
+			page.waitForNavigation( { waitUntil: 'networkidle0' } ),
+		] );
 		await page.waitForSelector(
 			'.widget select[name=currency] option[value=EUR][selected]'
 		);