From 5cd1512ec088ada0bba101ae0ef0df81e4333649 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Fri, 17 Nov 2023 17:04:45 +0100
Subject: [PATCH 01/51] Run performance tests in a matrix

---
 .github/workflows/performance.yml             | 49 ++++++-------
 tests/performance/compare-results.js          | 68 ++++++++-----------
 .../config/performance-reporter.js            |  6 +-
 tests/performance/results.js                  | 42 ------------
 tests/performance/specs/admin-l10n.test.js    | 52 --------------
 tests/performance/specs/admin.test.js         |  4 --
 .../specs/home-block-theme-l10n.test.js       | 63 -----------------
 .../specs/home-block-theme.test.js            | 57 ----------------
 .../specs/home-classic-theme-l10n.test.js     | 62 -----------------
 ...ome-classic-theme.test.js => home.test.js} |  8 +--
 10 files changed, 61 insertions(+), 350 deletions(-)
 delete mode 100644 tests/performance/results.js
 delete mode 100644 tests/performance/specs/admin-l10n.test.js
 delete mode 100644 tests/performance/specs/home-block-theme-l10n.test.js
 delete mode 100644 tests/performance/specs/home-block-theme.test.js
 delete mode 100644 tests/performance/specs/home-classic-theme-l10n.test.js
 rename tests/performance/specs/{home-classic-theme.test.js => home.test.js} (84%)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index ce7767e57d8fa..b402104d726b6 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -88,11 +88,17 @@ jobs:
   # - Ensure version-controlled files are not modified or deleted.
   # - Dispatch workflow run.
   performance:
-    name: Run performance tests
+    name: Run performance tests / ${{ matrix.locale }}) / ${{ matrix.theme }} ${{ matrix.memcached && ' with memcached' || '' }}
     runs-on: ubuntu-latest
     permissions:
       contents: read
     if: ${{ ( github.repository == 'WordPress/wordpress-develop' || github.event_name == 'pull_request' ) && ! contains( github.event.before, '00000000' ) }}
+    strategy:
+      fail-fast: false
+      matrix:
+        theme: [ 'twentytwentyone', 'twentytwentythree' ]
+        memcached: [ true, false ]
+        locale: [ 'en_US', 'de_DE' ]
 
     steps:
       - name: Configure environment variables
@@ -130,8 +136,9 @@ jobs:
         run: npm run build
 
       - name: Start Docker environment
-        run: |
-          npm run env:start
+        run: npm run env:start
+        env:
+          LOCAL_PHP_MEMCACHED: ${{ matrix.memcached }}
 
       - name: Log running Docker containers
         run: docker ps -a
@@ -162,11 +169,17 @@ jobs:
         run: |
           npm run env:cli -- rewrite structure '/%year%/%monthnum%/%postname%/' --path=/var/www/${{ env.LOCAL_DIR }}
 
-      - name: Install additional languages
+      - name: Configure theme
+        run: |
+          npm run env:cli -- theme activate ${{ matrix.theme }} --path=/var/www/${{ env.LOCAL_DIR }}
+
+      - name: Configure language
+        if: ${{ matrix.locale != 'en_US }}
         run: |
-          npm run env:cli -- language core install de_DE --path=/var/www/${{ env.LOCAL_DIR }}
-          npm run env:cli -- language plugin install de_DE --all --path=/var/www/${{ env.LOCAL_DIR }}
-          npm run env:cli -- language theme install de_DE --all --path=/var/www/${{ env.LOCAL_DIR }}
+          npm run env:cli -- language core install ${{ matrix.locale }} --path=/var/www/${{ env.LOCAL_DIR }}
+          npm run env:cli -- language plugin install ${{ matrix.locale }} --all --path=/var/www/${{ env.LOCAL_DIR }}
+          npm run env:cli -- language theme install ${{ matrix.locale }} --all --path=/var/www/${{ env.LOCAL_DIR }}
+          npm run env:cli -- site switch-language ${{ matrix.locale }}
 
       - name: Install MU plugin
         run: |
@@ -176,9 +189,6 @@ jobs:
       - name: Run performance tests (current commit)
         run: npm run test:performance
 
-      - name: Print performance tests results
-        run: node ./tests/performance/results.js
-
       - name: Check out target commit (target branch or previous commit)
         run: |
           if [[ -z "$TARGET_REF" ]]; then
@@ -194,16 +204,14 @@ jobs:
       - name: Build WordPress
         run: npm run build
 
+      - name: Flush cache
+        run: npm run env:cli -- cache flush --path=/var/www/${{ env.LOCAL_DIR }}
+
       - name: Run target performance tests (base/previous commit)
         env:
           TEST_RESULTS_PREFIX: before
         run: npm run test:performance
 
-      - name: Print target performance tests results
-        env:
-          TEST_RESULTS_PREFIX: before
-        run: node ./tests/performance/results.js
-
       - name: Reset to original commit
         run: git reset --hard $GITHUB_SHA
 
@@ -215,22 +223,17 @@ jobs:
           npm run env:cli -- core update --version=${{ env.BASE_TAG }} --force --path=/var/www/${{ env.LOCAL_DIR }}
           npm run env:cli -- core version --path=/var/www/${{ env.LOCAL_DIR }}
 
+      - name: Flush cache
+        run: npm run env:cli -- cache flush --path=/var/www/${{ env.LOCAL_DIR }}
+
       - name: Run baseline performance tests
         env:
           TEST_RESULTS_PREFIX: base
         run: npm run test:performance
 
-      - name: Print baseline performance tests results
-        env:
-          TEST_RESULTS_PREFIX: base
-        run: node ./tests/performance/results.js
-
       - name: Compare results with base
         run: node ./tests/performance/compare-results.js ${{ runner.temp }}/summary.md
 
-      - name: Add workflow summary
-        run: cat ${{ runner.temp }}/summary.md >> $GITHUB_STEP_SUMMARY
-
       - name: Set the base sha
         # Only needed when publishing results.
         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
diff --git a/tests/performance/compare-results.js b/tests/performance/compare-results.js
index 6af85a2f122f8..bb1d82532d132 100644
--- a/tests/performance/compare-results.js
+++ b/tests/performance/compare-results.js
@@ -3,14 +3,19 @@
 /**
  * External dependencies.
  */
-const fs = require( 'node:fs' );
-const path = require( 'node:path' );
+const { readFileSync, readDirSync } = require( 'node:fs' );
+const { join, basename } = require( 'node:path' );
 
 /**
  * Internal dependencies
  */
 const { median } = require( './utils' );
 
+process.env.WP_ARTIFACTS_PATH ??= join( process.cwd(), 'artifacts' );
+
+const args = process.argv.slice( 2 );
+const summaryFile = args[ 0 ];
+
 /**
  * Parse test files into JSON objects.
  *
@@ -19,36 +24,25 @@ const { median } = require( './utils' );
  */
 const parseFile = ( fileName ) =>
 	JSON.parse(
-		fs.readFileSync( path.join( __dirname, '/specs/', fileName ), 'utf8' )
+		readFileSync( join( process.env.WP_ARTIFACTS_PATH, fileName ), 'utf8' )
 	);
 
-// The list of test suites to log.
-const testSuites = [
-	'admin',
-	'admin-l10n',
-	'home-block-theme',
-	'home-block-theme-l10n',
-	'home-classic-theme',
-	'home-classic-theme-l10n',
-];
-
-// The current commit's results.
-const testResults = Object.fromEntries(
-	testSuites
-		.filter( ( key ) => fs.existsSync( path.join( __dirname, '/specs/', `${ key }.test.results.json` ) ) )
-		.map( ( key ) => [ key, parseFile( `${ key }.test.results.json` ) ] )
-);
-
-// The previous commit's results.
-const prevResults = Object.fromEntries(
-	testSuites
-		.filter( ( key ) => fs.existsSync( path.join( __dirname, '/specs/', `before-${ key }.test.results.json` ) ) )
-		.map( ( key ) => [ key, parseFile( `before-${ key }.test.results.json` ) ] )
-);
+const testResults = {};
+const prevResults = {};
 
-const args = process.argv.slice( 2 );
+for ( const { name } of readDirSync( process.env.WP_ARTIFACTS_PATH ) ) {
+	if ( ! name.endsWith( '.results.json' ) ) {
+		continue;
+	}
 
-const summaryFile = args[ 0 ];
+	const testSuiteName = basename( name, '.results.json' );
+
+	if ( ! name.startsWith( 'before-' ) ) {
+		testResults[ testSuiteName ] = parseFile( name );
+	} else {
+		prevResults[ testSuiteName ] = parseFile( name );
+	}
+}
 
 /**
  * Formats an array of objects as a Markdown table.
@@ -115,16 +109,14 @@ function linkToSha(sha) {
 	return `[${sha.slice(0, 7)}](https://github.com/${repoName}/commit/${sha})`;
 }
 
-let summaryMarkdown = `# Performance Test Results\n\n`;
-
-if ( process.env.GITHUB_SHA ) {
-	summaryMarkdown += `🛎️ Performance test results for ${ linkToSha( process.env.GITHUB_SHA ) } are in!\n\n`;
-} else {
-	summaryMarkdown += `🛎️ Performance test results are in!\n\n`;
-}
+let summaryMarkdown = `## Performance Test Results\n\n`;
 
 if ( process.env.TARGET_SHA ) {
-	summaryMarkdown += `This compares the results from this commit with the ones from ${ linkToSha( process.env.TARGET_SHA ) }.\n\n`;
+	if ( process.env.GITHUB_SHA ) {
+		summaryMarkdown += `This compares the results from this commit (${ linkToSha( process.env.GITHUB_SHA ) }) with the ones from ${linkToSha(process.env.TARGET_SHA)}.\n\n`;
+	} else {
+		summaryMarkdown += `This compares the results from this commit with the ones from ${linkToSha(process.env.TARGET_SHA)}.\n\n`;
+	}
 }
 
 if ( process.env.GITHUB_SHA ) {
@@ -179,7 +171,7 @@ for ( const key of testSuites ) {
 	}
 
 	if ( rows.length > 0 ) {
-		summaryMarkdown += `## ${ title }\n\n`;
+		summaryMarkdown += `### ${ title }\n\n`;
 		summaryMarkdown += `${ formatAsMarkdownTable( rows ) }\n`;
 
 		console.log( title );
@@ -188,7 +180,7 @@ for ( const key of testSuites ) {
 }
 
 if ( summaryFile ) {
-	fs.writeFileSync(
+	writeFileSync(
 		summaryFile,
 		summaryMarkdown
 	);
diff --git a/tests/performance/config/performance-reporter.js b/tests/performance/config/performance-reporter.js
index e557faa135cbd..a617fcbe343fc 100644
--- a/tests/performance/config/performance-reporter.js
+++ b/tests/performance/config/performance-reporter.js
@@ -1,7 +1,7 @@
 /**
  * External dependencies
  */
-import { join, dirname, basename } from 'node:path';
+import { join, basename } from 'node:path';
 import { writeFileSync } from 'node:fs';
 
 /**
@@ -26,8 +26,8 @@ class PerformanceReporter {
 		if ( performanceResults?.body ) {
 			writeFileSync(
 				join(
-					dirname( test.location.file ),
-					getResultsFilename( basename( test.location.file, '.js' ) )
+					process.env.WP_ARTIFACTS_PATH,
+					getResultsFilename( basename( test.location.file, '.test.js' ) )
 				),
 				performanceResults.body.toString( 'utf-8' )
 			);
diff --git a/tests/performance/results.js b/tests/performance/results.js
deleted file mode 100644
index d9f981f5e7a0e..0000000000000
--- a/tests/performance/results.js
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * External dependencies.
- */
-const fs = require( 'node:fs' );
-const { join } = require( 'node:path' );
-const { median, getResultsFilename } = require( './utils' );
-
-const testSuites = [
-    'admin',
-    'admin-l10n',
-    'home-classic-theme',
-    'home-classic-theme-l10n',
-    'home-block-theme',
-    'home-block-theme-l10n',
-];
-
-console.log( '\n>> 🎉 Results 🎉 \n' );
-
-for ( const testSuite of testSuites ) {
-    const resultsFileName = getResultsFilename( testSuite + '.test' );
-    const resultsPath = join( __dirname, '/specs/', resultsFileName );
-    fs.readFile( resultsPath, "utf8", ( err, data ) => {
-        if ( err ) {
-            console.log( "File read failed:", err );
-            return;
-        }
-        const convertString = testSuite.charAt( 0 ).toUpperCase() + testSuite.slice( 1 );
-        console.log( convertString.replace( /[-]+/g, " " ) + ':' );
-
-        tableData = JSON.parse( data );
-        const rawResults = [];
-
-        for ( var key in tableData ) {
-            if ( tableData.hasOwnProperty( key ) ) {
-                rawResults[ key ] = median( tableData[ key ] );
-            }
-        }
-        console.table( rawResults );
-    });
-}
diff --git a/tests/performance/specs/admin-l10n.test.js b/tests/performance/specs/admin-l10n.test.js
deleted file mode 100644
index a8c9be09975a8..0000000000000
--- a/tests/performance/specs/admin-l10n.test.js
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * WordPress dependencies
- */
-import { test } from '@wordpress/e2e-test-utils-playwright';
-
-/**
- * Internal dependencies
- */
-import { camelCaseDashes } from '../utils';
-
-const results = {
-	timeToFirstByte: [],
-};
-
-test.describe( 'Admin (L10N)', () => {
-	test.beforeAll( async ( { requestUtils } ) => {
-		await requestUtils.activateTheme( 'twentytwentyone' );
-		await requestUtils.updateSiteSettings( {
-			language: 'de_DE',
-		} );
-	} );
-
-	test.afterAll( async ( { requestUtils }, testInfo ) => {
-		await testInfo.attach( 'results', {
-			body: JSON.stringify( results, null, 2 ),
-			contentType: 'application/json',
-		} );
-		await requestUtils.updateSiteSettings( {
-			language: '',
-		} );
-	} );
-
-	const iterations = Number( process.env.TEST_RUNS );
-	for ( let i = 1; i <= iterations; i++ ) {
-		test( `Measure load time metrics (${ i } of ${ iterations })`, async ( {
-			admin,
-			metrics,
-		} ) => {
-			await admin.visitAdminPage( '/' );
-
-			const serverTiming = await metrics.getServerTiming();
-
-			for ( const [ key, value ] of Object.entries( serverTiming ) ) {
-				results[ camelCaseDashes( key ) ] ??= [];
-				results[ camelCaseDashes( key ) ].push( value );
-			}
-
-			const ttfb = await metrics.getTimeToFirstByte();
-			results.timeToFirstByte.push( ttfb );
-		} );
-	}
-} );
diff --git a/tests/performance/specs/admin.test.js b/tests/performance/specs/admin.test.js
index 98602291142dc..3e64947afa668 100644
--- a/tests/performance/specs/admin.test.js
+++ b/tests/performance/specs/admin.test.js
@@ -13,10 +13,6 @@ const results = {
 };
 
 test.describe( 'Admin', () => {
-	test.beforeAll( async ( { requestUtils } ) => {
-		await requestUtils.activateTheme( 'twentytwentyone' );
-	} );
-
 	test.afterAll( async ( {}, testInfo ) => {
 		await testInfo.attach( 'results', {
 			body: JSON.stringify( results, null, 2 ),
diff --git a/tests/performance/specs/home-block-theme-l10n.test.js b/tests/performance/specs/home-block-theme-l10n.test.js
deleted file mode 100644
index 591925056ffc5..0000000000000
--- a/tests/performance/specs/home-block-theme-l10n.test.js
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * WordPress dependencies
- */
-import { test } from '@wordpress/e2e-test-utils-playwright';
-
-/**
- * Internal dependencies
- */
-import { camelCaseDashes } from '../utils';
-
-const results = {
-	timeToFirstByte: [],
-	largestContentfulPaint: [],
-	lcpMinusTtfb: [],
-};
-
-test.describe( 'Front End - Twenty Twenty Three (L10N)', () => {
-	test.use( {
-		storageState: {}, // User will be logged out.
-	} );
-
-	test.beforeAll( async ( { requestUtils } ) => {
-		await requestUtils.activateTheme( 'twentytwentythree' );
-		await requestUtils.updateSiteSettings( {
-			language: 'de_DE',
-		} );
-	} );
-
-	test.afterAll( async ( { requestUtils }, testInfo ) => {
-		await testInfo.attach( 'results', {
-			body: JSON.stringify( results, null, 2 ),
-			contentType: 'application/json',
-		} );
-		await requestUtils.activateTheme( 'twentytwentyone' );
-		await requestUtils.updateSiteSettings( {
-			language: '',
-		} );
-	} );
-
-	const iterations = Number( process.env.TEST_RUNS );
-	for ( let i = 1; i <= iterations; i++ ) {
-		test( `Measure load time metrics (${ i } of ${ iterations })`, async ( {
-			page,
-			metrics,
-		} ) => {
-			await page.goto( '/' );
-
-			const serverTiming = await metrics.getServerTiming();
-
-			for ( const [ key, value ] of Object.entries( serverTiming ) ) {
-				results[ camelCaseDashes( key ) ] ??= [];
-				results[ camelCaseDashes( key ) ].push( value );
-			}
-
-			const ttfb = await metrics.getTimeToFirstByte();
-			const lcp = await metrics.getLargestContentfulPaint();
-
-			results.largestContentfulPaint.push( lcp );
-			results.timeToFirstByte.push( ttfb );
-			results.lcpMinusTtfb.push( lcp - ttfb );
-		} );
-	}
-} );
diff --git a/tests/performance/specs/home-block-theme.test.js b/tests/performance/specs/home-block-theme.test.js
deleted file mode 100644
index 00bccc6996e35..0000000000000
--- a/tests/performance/specs/home-block-theme.test.js
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * WordPress dependencies
- */
-import { test } from '@wordpress/e2e-test-utils-playwright';
-
-/**
- * Internal dependencies
- */
-import { camelCaseDashes } from '../utils';
-
-const results = {
-	timeToFirstByte: [],
-	largestContentfulPaint: [],
-	lcpMinusTtfb: [],
-};
-
-test.describe( 'Front End - Twenty Twenty Three', () => {
-	test.use( {
-		storageState: {}, // User will be logged out.
-	} );
-
-	test.beforeAll( async ( { requestUtils } ) => {
-		await requestUtils.activateTheme( 'twentytwentythree' );
-	} );
-
-	test.afterAll( async ( { requestUtils }, testInfo ) => {
-		await testInfo.attach( 'results', {
-			body: JSON.stringify( results, null, 2 ),
-			contentType: 'application/json',
-		} );
-		await requestUtils.activateTheme( 'twentytwentyone' );
-	} );
-
-	const iterations = Number( process.env.TEST_RUNS );
-	for ( let i = 1; i <= iterations; i++ ) {
-		test( `Measure load time metrics (${ i } of ${ iterations })`, async ( {
-			page,
-			metrics,
-		} ) => {
-			await page.goto( '/' );
-
-			const serverTiming = await metrics.getServerTiming();
-
-			for ( const [ key, value ] of Object.entries( serverTiming ) ) {
-				results[ camelCaseDashes( key ) ] ??= [];
-				results[ camelCaseDashes( key ) ].push( value );
-			}
-
-			const ttfb = await metrics.getTimeToFirstByte();
-			const lcp = await metrics.getLargestContentfulPaint();
-
-			results.largestContentfulPaint.push( lcp );
-			results.timeToFirstByte.push( ttfb );
-			results.lcpMinusTtfb.push( lcp - ttfb );
-		} );
-	}
-} );
diff --git a/tests/performance/specs/home-classic-theme-l10n.test.js b/tests/performance/specs/home-classic-theme-l10n.test.js
deleted file mode 100644
index e6f6e1cbb9818..0000000000000
--- a/tests/performance/specs/home-classic-theme-l10n.test.js
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * WordPress dependencies
- */
-import { test } from '@wordpress/e2e-test-utils-playwright';
-
-/**
- * Internal dependencies
- */
-import { camelCaseDashes } from '../utils';
-
-const results = {
-	timeToFirstByte: [],
-	largestContentfulPaint: [],
-	lcpMinusTtfb: [],
-};
-
-test.describe( 'Front End - Twenty Twenty One (L10N)', () => {
-	test.use( {
-		storageState: {}, // User will be logged out.
-	} );
-
-	test.beforeAll( async ( { requestUtils } ) => {
-		await requestUtils.activateTheme( 'twentytwentyone' );
-		await requestUtils.updateSiteSettings( {
-			language: 'de_DE',
-		} );
-	} );
-
-	test.afterAll( async ( { requestUtils }, testInfo ) => {
-		await testInfo.attach( 'results', {
-			body: JSON.stringify( results, null, 2 ),
-			contentType: 'application/json',
-		} );
-		await requestUtils.updateSiteSettings( {
-			language: '',
-		} );
-	} );
-
-	const iterations = Number( process.env.TEST_RUNS );
-	for ( let i = 1; i <= iterations; i++ ) {
-		test( `Measure load time metrics (${ i } of ${ iterations })`, async ( {
-			page,
-			metrics,
-		} ) => {
-			await page.goto( '/' );
-
-			const serverTiming = await metrics.getServerTiming();
-
-			for ( const [ key, value ] of Object.entries( serverTiming ) ) {
-				results[ camelCaseDashes( key ) ] ??= [];
-				results[ camelCaseDashes( key ) ].push( value );
-			}
-
-			const ttfb = await metrics.getTimeToFirstByte();
-			const lcp = await metrics.getLargestContentfulPaint();
-
-			results.largestContentfulPaint.push( lcp );
-			results.timeToFirstByte.push( ttfb );
-			results.lcpMinusTtfb.push( lcp - ttfb );
-		} );
-	}
-} );
diff --git a/tests/performance/specs/home-classic-theme.test.js b/tests/performance/specs/home.test.js
similarity index 84%
rename from tests/performance/specs/home-classic-theme.test.js
rename to tests/performance/specs/home.test.js
index a95e50fa06b9e..edbf98df16429 100644
--- a/tests/performance/specs/home-classic-theme.test.js
+++ b/tests/performance/specs/home.test.js
@@ -14,16 +14,12 @@ const results = {
 	lcpMinusTtfb: [],
 };
 
-test.describe( 'Front End - Twenty Twenty One', () => {
+test.describe( 'Front End', () => {
 	test.use( {
 		storageState: {}, // User will be logged out.
 	} );
 
-	test.beforeAll( async ( { requestUtils } ) => {
-		await requestUtils.activateTheme( 'twentytwentyone' );
-	} );
-
-	test.afterAll( async ( {}, testInfo ) => {
+	test.afterAll( async ( { requestUtils }, testInfo ) => {
 		await testInfo.attach( 'results', {
 			body: JSON.stringify( results, null, 2 ),
 			contentType: 'application/json',

From b117403b789af0895c646880039774049ca630fa Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Fri, 17 Nov 2023 17:16:16 +0100
Subject: [PATCH 02/51] Add missing quote

---
 .github/workflows/performance.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index b402104d726b6..ff546d7040f5c 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -174,7 +174,7 @@ jobs:
           npm run env:cli -- theme activate ${{ matrix.theme }} --path=/var/www/${{ env.LOCAL_DIR }}
 
       - name: Configure language
-        if: ${{ matrix.locale != 'en_US }}
+        if: ${{ matrix.locale != 'en_US' }}
         run: |
           npm run env:cli -- language core install ${{ matrix.locale }} --path=/var/www/${{ env.LOCAL_DIR }}
           npm run env:cli -- language plugin install ${{ matrix.locale }} --all --path=/var/www/${{ env.LOCAL_DIR }}

From fef9e380d841b266a4843c53787ccfe9e804a3ad Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Fri, 17 Nov 2023 17:25:44 +0100
Subject: [PATCH 03/51] install drop-in

---
 .github/workflows/performance.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index ff546d7040f5c..cafe286ed3ce7 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -140,6 +140,10 @@ jobs:
         env:
           LOCAL_PHP_MEMCACHED: ${{ matrix.memcached }}
 
+      - name: Install object cache drop-in
+        if: ${{ matrix.memcached }}
+        run: cp src/wp-content/object-cache.php build/wp-content/object-cache.php
+
       - name: Log running Docker containers
         run: docker ps -a
 

From c5f1180e32e2e4bccfca6540a6ea785a56a37cd3 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Fri, 17 Nov 2023 17:27:20 +0100
Subject: [PATCH 04/51] Add missing arg

---
 .github/workflows/performance.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index cafe286ed3ce7..00198b9c8d148 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -183,7 +183,7 @@ jobs:
           npm run env:cli -- language core install ${{ matrix.locale }} --path=/var/www/${{ env.LOCAL_DIR }}
           npm run env:cli -- language plugin install ${{ matrix.locale }} --all --path=/var/www/${{ env.LOCAL_DIR }}
           npm run env:cli -- language theme install ${{ matrix.locale }} --all --path=/var/www/${{ env.LOCAL_DIR }}
-          npm run env:cli -- site switch-language ${{ matrix.locale }}
+          npm run env:cli -- site switch-language ${{ matrix.locale }} --path=/var/www/${{ env.LOCAL_DIR }}
 
       - name: Install MU plugin
         run: |

From 5464464ddfac3d6103ca4168231a67048ec87d21 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Fri, 17 Nov 2023 17:29:36 +0100
Subject: [PATCH 05/51] Move up env var

---
 .github/workflows/performance.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index 00198b9c8d148..e97a9a65344f9 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -99,7 +99,8 @@ jobs:
         theme: [ 'twentytwentyone', 'twentytwentythree' ]
         memcached: [ true, false ]
         locale: [ 'en_US', 'de_DE' ]
-
+    env:
+      LOCAL_PHP_MEMCACHED: ${{ matrix.memcached }}
     steps:
       - name: Configure environment variables
         run: |
@@ -137,8 +138,6 @@ jobs:
 
       - name: Start Docker environment
         run: npm run env:start
-        env:
-          LOCAL_PHP_MEMCACHED: ${{ matrix.memcached }}
 
       - name: Install object cache drop-in
         if: ${{ matrix.memcached }}

From 538e6052d3ec000c9fd0b9efaa131bd1525304ef Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Fri, 17 Nov 2023 17:41:22 +0100
Subject: [PATCH 06/51] Fix function name

---
 tests/performance/compare-results.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/performance/compare-results.js b/tests/performance/compare-results.js
index bb1d82532d132..af4878c9bce24 100644
--- a/tests/performance/compare-results.js
+++ b/tests/performance/compare-results.js
@@ -3,7 +3,7 @@
 /**
  * External dependencies.
  */
-const { readFileSync, readDirSync } = require( 'node:fs' );
+const { readFileSync, readdirSync } = require( 'node:fs' );
 const { join, basename } = require( 'node:path' );
 
 /**
@@ -30,7 +30,7 @@ const parseFile = ( fileName ) =>
 const testResults = {};
 const prevResults = {};
 
-for ( const { name } of readDirSync( process.env.WP_ARTIFACTS_PATH ) ) {
+for ( const { name } of readdirSync( process.env.WP_ARTIFACTS_PATH ) ) {
 	if ( ! name.endsWith( '.results.json' ) ) {
 		continue;
 	}

From 109e7ab1e9abf83ab9a238809533d9f8ce0373a2 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Fri, 17 Nov 2023 17:53:51 +0100
Subject: [PATCH 07/51] Fix compare script logic

---
 tests/performance/compare-results.js | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tests/performance/compare-results.js b/tests/performance/compare-results.js
index af4878c9bce24..9f9d68fe4e41a 100644
--- a/tests/performance/compare-results.js
+++ b/tests/performance/compare-results.js
@@ -30,7 +30,7 @@ const parseFile = ( fileName ) =>
 const testResults = {};
 const prevResults = {};
 
-for ( const { name } of readdirSync( process.env.WP_ARTIFACTS_PATH ) ) {
+for ( const name of readdirSync( process.env.WP_ARTIFACTS_PATH ) ) {
 	if ( ! name.endsWith( '.results.json' ) ) {
 		continue;
 	}
@@ -144,8 +144,7 @@ function formatValue( metric, value) {
 	return `${ value.toFixed( 2 ) } ms`;
 }
 
-for ( const key of testSuites ) {
-	const current = testResults[ key ] || {};
+for ( const [ key, current ] of Object.entries( testResults ) ) {
 	const prev = prevResults[ key ] || {};
 
 	const title = ( key.charAt( 0 ).toUpperCase() + key.slice( 1 ) ).replace(

From d18523eb78e3bf471e995e9f64b0ea039a25d498 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Fri, 17 Nov 2023 18:02:12 +0100
Subject: [PATCH 08/51] Update name

---
 .github/workflows/performance.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index e97a9a65344f9..d46b438f4963c 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -88,7 +88,7 @@ jobs:
   # - Ensure version-controlled files are not modified or deleted.
   # - Dispatch workflow run.
   performance:
-    name: Run performance tests / ${{ matrix.locale }}) / ${{ matrix.theme }} ${{ matrix.memcached && ' with memcached' || '' }}
+    name: ${{ matrix.locale }} / ${{ matrix.theme }} ${{ matrix.memcached && ' with memcached' || '' }}
     runs-on: ubuntu-latest
     permissions:
       contents: read

From b762b79b8ad048dd3cd3d9befae16502987ddf2f Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Fri, 17 Nov 2023 18:03:53 +0100
Subject: [PATCH 09/51] Add missing import

---
 tests/performance/compare-results.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/performance/compare-results.js b/tests/performance/compare-results.js
index 9f9d68fe4e41a..7340ee74a7f1f 100644
--- a/tests/performance/compare-results.js
+++ b/tests/performance/compare-results.js
@@ -3,7 +3,7 @@
 /**
  * External dependencies.
  */
-const { readFileSync, readdirSync } = require( 'node:fs' );
+const { readFileSync, readdirSync, writeFileSync } = require( 'node:fs' );
 const { join, basename } = require( 'node:path' );
 
 /**

From 782b4c63e1fa76948cff774107362435f1aaa36a Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Fri, 17 Nov 2023 18:34:55 +0100
Subject: [PATCH 10/51] Add back the summary

---
 .github/workflows/performance.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index d46b438f4963c..c782f2676dc33 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -237,6 +237,9 @@ jobs:
       - name: Compare results with base
         run: node ./tests/performance/compare-results.js ${{ runner.temp }}/summary.md
 
+      - name: Add workflow summary
+        run: cat ${{ runner.temp }}/summary.md >> $GITHUB_STEP_SUMMARY
+
       - name: Set the base sha
         # Only needed when publishing results.
         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}

From 667bfebbe0beaca6361be7e1d001fcf8dcf37f33 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Fri, 17 Nov 2023 19:42:53 +0100
Subject: [PATCH 11/51] Skip base files

---
 tests/performance/compare-results.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/performance/compare-results.js b/tests/performance/compare-results.js
index 7340ee74a7f1f..295b59d683496 100644
--- a/tests/performance/compare-results.js
+++ b/tests/performance/compare-results.js
@@ -31,7 +31,7 @@ const testResults = {};
 const prevResults = {};
 
 for ( const name of readdirSync( process.env.WP_ARTIFACTS_PATH ) ) {
-	if ( ! name.endsWith( '.results.json' ) ) {
+	if ( ! name.endsWith( '.results.json' ) || ! name.startsWith( 'base-' ) ) {
 		continue;
 	}
 

From 1dc70bd7ea7c539aae539c76e077522071ed67c8 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Fri, 26 Apr 2024 16:51:30 +0200
Subject: [PATCH 12/51] Move back some logic to JS

---
 .github/workflows/performance.yml     | 18 ++-----
 tests/performance/specs/admin.test.js | 68 +++++++++++++++++-------
 tests/performance/specs/home.test.js  | 75 ++++++++++++++++++---------
 3 files changed, 104 insertions(+), 57 deletions(-)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index b0ad639ebff04..46303b3a46e31 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -92,7 +92,7 @@ jobs:
   # - Ensure version-controlled files are not modified or deleted.
   # - Dispatch workflow run.
   performance:
-    name: ${{ matrix.locale }} / ${{ matrix.theme }} ${{ matrix.memcached && ' with memcached' || '' }}
+    name: Run performance tests ${{ matrix.memcached && '(with memcached)' || '' }}
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -100,9 +100,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        theme: [ 'twentytwentyone', 'twentytwentythree' ]
         memcached: [ true, false ]
-        locale: [ 'en_US', 'de_DE' ]
     env:
       LOCAL_PHP_MEMCACHED: ${{ matrix.memcached }}
     steps:
@@ -174,17 +172,11 @@ jobs:
         run: |
           npm run env:cli -- rewrite structure '/%year%/%monthnum%/%postname%/' --path=/var/www/${{ env.LOCAL_DIR }}
 
-      - name: Configure theme
+      - name: Install additional languages
         run: |
-          npm run env:cli -- theme activate ${{ matrix.theme }} --path=/var/www/${{ env.LOCAL_DIR }}
-
-      - name: Configure language
-        if: ${{ matrix.locale != 'en_US' }}
-        run: |
-          npm run env:cli -- language core install ${{ matrix.locale }} --path=/var/www/${{ env.LOCAL_DIR }}
-          npm run env:cli -- language plugin install ${{ matrix.locale }} --all --path=/var/www/${{ env.LOCAL_DIR }}
-          npm run env:cli -- language theme install ${{ matrix.locale }} --all --path=/var/www/${{ env.LOCAL_DIR }}
-          npm run env:cli -- site switch-language ${{ matrix.locale }} --path=/var/www/${{ env.LOCAL_DIR }}
+          npm run env:cli -- language core install de_DE --path=/var/www/${{ env.LOCAL_DIR }}
+          npm run env:cli -- language plugin install de_DE --all --path=/var/www/${{ env.LOCAL_DIR }}
+          npm run env:cli -- language theme install de_DE --all --path=/var/www/${{ env.LOCAL_DIR }}
 
       - name: Install MU plugin
         run: |
diff --git a/tests/performance/specs/admin.test.js b/tests/performance/specs/admin.test.js
index 3e64947afa668..36bdd9c628aa1 100644
--- a/tests/performance/specs/admin.test.js
+++ b/tests/performance/specs/admin.test.js
@@ -12,31 +12,59 @@ const results = {
 	timeToFirstByte: [],
 };
 
+const locales = [ 'en_US', 'de_DE' ];
+
 test.describe( 'Admin', () => {
-	test.afterAll( async ( {}, testInfo ) => {
-		await testInfo.attach( 'results', {
-			body: JSON.stringify( results, null, 2 ),
-			contentType: 'application/json',
-		} );
-	} );
+	for ( const locale of locales ) {
+		test.describe( `Locale: ${ locale }`, () => {
+			test.beforeAll( async ( { requestUtils } ) => {
+				await requestUtils.activateTheme( 'twentytwentyone' );
+				await requestUtils.updateSiteSettings( {
+					language: 'en_US' === locale ? '' : locale,
+				} );
+			} );
 
-	const iterations = Number( process.env.TEST_RUNS );
-	for ( let i = 1; i <= iterations; i++ ) {
-		test( `Measure load time metrics (${ i } of ${ iterations })`, async ( {
-			admin,
-			metrics,
-		} ) => {
-			await admin.visitAdminPage( '/' );
+			test.afterAll( async ( { requestUtils }, testInfo ) => {
+				await testInfo.attach( 'results', {
+					body: JSON.stringify( results, null, 2 ),
+					contentType: 'application/json',
+				} );
 
-			const serverTiming = await metrics.getServerTiming();
+				await requestUtils.updateSiteSettings( {
+					language: '',
+				} );
 
-			for ( const [ key, value ] of Object.entries( serverTiming ) ) {
-				results[ camelCaseDashes( key ) ] ??= [];
-				results[ camelCaseDashes( key ) ].push( value );
-			}
+				results.timeToFirstByte = [];
+			} );
+
+			test.afterAll( async ( {}, testInfo ) => {
+				await testInfo.attach( 'results', {
+					body: JSON.stringify( results, null, 2 ),
+					contentType: 'application/json',
+				} );
+			} );
 
-			const ttfb = await metrics.getTimeToFirstByte();
-			results.timeToFirstByte.push( ttfb );
+			const iterations = Number( process.env.TEST_RUNS );
+			for ( let i = 1; i <= iterations; i++ ) {
+				test( `Measure load time metrics (${ i } of ${ iterations })`, async ( {
+					admin,
+					metrics,
+				} ) => {
+					await admin.visitAdminPage( '/' );
+
+					const serverTiming = await metrics.getServerTiming();
+
+					for ( const [ key, value ] of Object.entries(
+						serverTiming
+					) ) {
+						results[ camelCaseDashes( key ) ] ??= [];
+						results[ camelCaseDashes( key ) ].push( value );
+					}
+
+					const ttfb = await metrics.getTimeToFirstByte();
+					results.timeToFirstByte.push( ttfb );
+				} );
+			}
 		} );
 	}
 } );
diff --git a/tests/performance/specs/home.test.js b/tests/performance/specs/home.test.js
index edbf98df16429..b88b6adb9f362 100644
--- a/tests/performance/specs/home.test.js
+++ b/tests/performance/specs/home.test.js
@@ -14,39 +14,66 @@ const results = {
 	lcpMinusTtfb: [],
 };
 
+const themes = [ 'twentytwentyone', 'twentytwentythree', 'twentytwentyfour' ];
+
+const locales = [ 'en_US', 'de_DE' ];
+
 test.describe( 'Front End', () => {
 	test.use( {
 		storageState: {}, // User will be logged out.
 	} );
 
-	test.afterAll( async ( { requestUtils }, testInfo ) => {
-		await testInfo.attach( 'results', {
-			body: JSON.stringify( results, null, 2 ),
-			contentType: 'application/json',
-		} );
-	} );
+	for ( const theme of themes ) {
+		for ( const locale of locales ) {
+			test.describe( `Theme: ${ theme }, Locale: ${ locale }`, () => {
+				test.beforeAll( async ( { requestUtils } ) => {
+					await requestUtils.activateTheme( theme );
+					await requestUtils.updateSiteSettings( {
+						language: 'en_US' === locale ? '' : locale,
+					} );
+				} );
+
+				test.afterAll( async ( { requestUtils }, testInfo ) => {
+					await testInfo.attach( 'results', {
+						body: JSON.stringify( results, null, 2 ),
+						contentType: 'application/json',
+					} );
+
+					await requestUtils.updateSiteSettings( {
+						language: '',
+					} );
+
+					results.largestContentfulPaint = [];
+					results.timeToFirstByte = [];
+					results.lcpMinusTtfb = [];
+				} );
 
-	const iterations = Number( process.env.TEST_RUNS );
-	for ( let i = 1; i <= iterations; i++ ) {
-		test( `Measure load time metrics (${ i } of ${ iterations })`, async ( {
-			page,
-			metrics,
-		} ) => {
-			await page.goto( '/' );
+				const iterations = Number( process.env.TEST_RUNS );
+				for ( let i = 1; i <= iterations; i++ ) {
+					test( `Measure load time metrics (${ i } of ${ iterations })`, async ( {
+						page,
+						metrics,
+					} ) => {
+						await page.goto( '/' );
 
-			const serverTiming = await metrics.getServerTiming();
+						const serverTiming = await metrics.getServerTiming();
 
-			for ( const [ key, value ] of Object.entries( serverTiming ) ) {
-				results[ camelCaseDashes( key ) ] ??= [];
-				results[ camelCaseDashes( key ) ].push( value );
-			}
+						for ( const [ key, value ] of Object.entries(
+							serverTiming
+						) ) {
+							results[ camelCaseDashes( key ) ] ??= [];
+							results[ camelCaseDashes( key ) ].push( value );
+						}
 
-			const ttfb = await metrics.getTimeToFirstByte();
-			const lcp = await metrics.getLargestContentfulPaint();
+						const ttfb = await metrics.getTimeToFirstByte();
+						const lcp = await metrics.getLargestContentfulPaint();
 
-			results.largestContentfulPaint.push( lcp );
-			results.timeToFirstByte.push( ttfb );
-			results.lcpMinusTtfb.push( lcp - ttfb );
-		} );
+						results.largestContentfulPaint.push( lcp );
+						results.timeToFirstByte.push( ttfb );
+						results.lcpMinusTtfb.push( lcp - ttfb );
+					} );
+				}
+			} );
+		}
 	}
 } );

From 569e2db89a07e59a246feb5591b86dc079aa3eec Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Fri, 26 Apr 2024 23:28:19 +0200
Subject: [PATCH 13/51] Update scripts

---
 tests/performance/compare-results.js          | 196 ++++++------------
 tests/performance/config/global-setup.js      |   4 +-
 .../config/performance-reporter.js            |  68 ++++--
 tests/performance/log-results.js              |  43 ++--
 tests/performance/playwright.config.js        |   1 -
 tests/performance/utils.js                    |  99 +++++++--
 6 files changed, 228 insertions(+), 183 deletions(-)

diff --git a/tests/performance/compare-results.js b/tests/performance/compare-results.js
index 295b59d683496..2e6af694953fa 100644
--- a/tests/performance/compare-results.js
+++ b/tests/performance/compare-results.js
@@ -3,13 +3,18 @@
 /**
  * External dependencies.
  */
-const { readFileSync, readdirSync, writeFileSync } = require( 'node:fs' );
-const { join, basename } = require( 'node:path' );
+const { readFileSync, writeFileSync } = require( 'node:fs' );
+const { join, existsSync } = require( 'node:path' );
 
 /**
  * Internal dependencies
  */
-const { median } = require( './utils' );
+const {
+	median,
+	formatAsMarkdownTable,
+	formatValue,
+	linkToSha,
+} = require( './utils' );
 
 process.env.WP_ARTIFACTS_PATH ??= join( process.cwd(), 'artifacts' );
 
@@ -20,102 +25,38 @@ const summaryFile = args[ 0 ];
  * Parse test files into JSON objects.
  *
  * @param {string} fileName The name of the file.
- * @returns An array of parsed objects from each file.
+ * @return {Array<{file: string, title: string, results: Record<string,number[]>[]}>} Parsed object.
  */
-const parseFile = ( fileName ) =>
-	JSON.parse(
-		readFileSync( join( process.env.WP_ARTIFACTS_PATH, fileName ), 'utf8' )
-	);
-
-const testResults = {};
-const prevResults = {};
-
-for ( const name of readdirSync( process.env.WP_ARTIFACTS_PATH ) ) {
-	if ( ! name.endsWith( '.results.json' ) || ! name.startsWith( 'base-' ) ) {
-		continue;
+function parseFile( fileName ) {
+	const file = join( process.env.WP_ARTIFACTS_PATH, fileName );
+	if ( ! existsSync( file ) ) {
+		return [];
 	}
 
-	const testSuiteName = basename( name, '.results.json' );
-
-	if ( ! name.startsWith( 'before-' ) ) {
-		testResults[ testSuiteName ] = parseFile( name );
-	} else {
-		prevResults[ testSuiteName ] = parseFile( name );
-	}
+	return JSON.parse( readFileSync( file, 'utf8' ) );
 }
 
 /**
- * Formats an array of objects as a Markdown table.
- *
- * For example, this array:
- *
- * [
- * 	{
- * 	    foo: 123,
- * 	    bar: 456,
- * 	    baz: 'Yes',
- * 	},
- * 	{
- * 	    foo: 777,
- * 	    bar: 999,
- * 	    baz: 'No',
- * 	}
- * ]
- *
- * Will result in the following table:
- *
- * | foo | bar | baz |
- * |-----|-----|-----|
- * | 123 | 456 | Yes |
- * | 777 | 999 | No  |
- *
- * @param {Array<Object>} rows Table rows.
- * @returns {string} Markdown table content.
+ * @type {Array<{file: string, title: string, results: Record<string,number[]>[]}>}
  */
-function formatAsMarkdownTable( rows ) {
-	let result = '';
-	const headers = Object.keys( rows[ 0 ] );
-	for ( const header of headers ) {
-		result += `| ${ header } `;
-	}
-	result += '|\n';
-	for ( const header of headers ) {
-		result += '| ------ ';
-	}
-	result += '|\n';
-
-	for ( const row of rows ) {
-		for ( const value of Object.values( row ) ) {
-			result += `| ${ value } `;
-		}
-		result += '|\n';
-	}
-
-	return result;
-}
+const beforeStats = parseFile( 'before-performance-results.json' );
 
 /**
- * Returns a Markdown link to a Git commit on the current GitHub repository.
- *
- * For example, turns `a5c3785ed8d6a35868bc169f07e40e889087fd2e`
- * into (https://github.com/wordpress/wordpress-develop/commit/36fe58a8c64dcc83fc21bddd5fcf054aef4efb27)[36fe58a].
- *
- * @param {string} sha Commit SHA.
- * @return string Link
+ * @type {Array<{file: string, title: string, results: Record<string,number[]>[]}>}
  */
-function linkToSha(sha) {
-	const repoName = process.env.GITHUB_REPOSITORY || 'wordpress/wordpress-develop';
-
-	return `[${sha.slice(0, 7)}](https://github.com/${repoName}/commit/${sha})`;
-}
+const afterStats = parseFile( 'performance-results.json' );
 
 let summaryMarkdown = `## Performance Test Results\n\n`;
 
 if ( process.env.TARGET_SHA ) {
 	if ( process.env.GITHUB_SHA ) {
-		summaryMarkdown += `This compares the results from this commit (${ linkToSha( process.env.GITHUB_SHA ) }) with the ones from ${linkToSha(process.env.TARGET_SHA)}.\n\n`;
+		summaryMarkdown += `This compares the results from this commit (${ linkToSha(
+			process.env.GITHUB_SHA
+		) }) with the ones from ${ linkToSha( process.env.TARGET_SHA ) }.\n\n`;
 	} else {
-		summaryMarkdown += `This compares the results from this commit with the ones from ${linkToSha(process.env.TARGET_SHA)}.\n\n`;
+		summaryMarkdown += `This compares the results from this commit with the ones from ${ linkToSha(
+			process.env.TARGET_SHA
+		) }.\n\n`;
 	}
 }
 
@@ -125,62 +66,55 @@ if ( process.env.GITHUB_SHA ) {
 
 console.log( 'Performance Test Results\n' );
 
-console.log( 'Note: Due to the nature of how GitHub Actions work, some variance in the results is expected.\n' );
-
-/**
- * Nicely formats a given value.
- *
- * @param {string} metric Metric.
- * @param {number} value
- */
-function formatValue( metric, value) {
-	if ( null === value ) {
-		return 'N/A';
-	}
-	if ( 'wpMemoryUsage' === metric ) {
-		return `${ ( value / Math.pow( 10, 6 ) ).toFixed( 2 ) } MB`;
-	}
-
-	return `${ value.toFixed( 2 ) } ms`;
-}
-
-for ( const [ key, current ] of Object.entries( testResults ) ) {
-	const prev = prevResults[ key ] || {};
+console.log(
+	'Note: Due to the nature of how GitHub Actions work, some variance in the results is expected.\n'
+);
 
-	const title = ( key.charAt( 0 ).toUpperCase() + key.slice( 1 ) ).replace(
-		/-+/g,
-		' '
-	);
+for ( const { file, title, results } of afterStats ) {
+	const prevStat = beforeStats.find( ( s ) => s.file === file );
 
+	/**
+	 * @type {Array<Record<string, string>>}
+	 */
 	const rows = [];
 
-	for ( const [ metric, values ] of Object.entries( current ) ) {
-		const value = median( values );
-		const prevValue = prev[ metric ] ? median( prev[ metric ] ) : null;
-
-		const delta = null !== prevValue ? value - prevValue : 0
-		const percentage = ( delta / value ) * 100;
-		rows.push( {
-			Metric: metric,
-			Before: formatValue( metric, prevValue ),
-			After: formatValue( metric, value ),
-			'Diff abs.': formatValue( metric, delta ),
-			'Diff %': `${ percentage.toFixed( 2 ) } %`,
-		} );
+	for ( const i in results ) {
+		const newResult = results[ i ];
+
+		for ( const [ metric, values ] of Object.entries( newResult ) ) {
+			// Only do comparison if the number of results is the same.
+			const prevValues =
+				prevStat?.results.length === results.length
+					? prevStat?.results[ i ].key
+					: null;
+
+			const value = median( values );
+			const prevValue = prevValues ? median( prevValues ) : 0;
+			const delta = value - prevValue;
+			const percentage = ( delta / value ) * 100;
+
+			rows.push( {
+				Metric: metric,
+				Before: formatValue( metric, prevValue ),
+				After: formatValue( metric, value ),
+				'Diff abs.': formatValue( metric, delta ),
+				'Diff %': `${ percentage.toFixed( 2 ) } %`,
+			} );
+		}
 	}
 
-	if ( rows.length > 0 ) {
-		summaryMarkdown += `### ${ title }\n\n`;
-		summaryMarkdown += `${ formatAsMarkdownTable( rows ) }\n`;
+	console.log( title );
+	console.table( rows );
 
-		console.log( title );
-		console.table( rows );
-	}
+	summaryMarkdown += `**${ title }**\n\n`;
+	summaryMarkdown += `${ formatAsMarkdownTable( rows ) }\n`;
 }
 
+writeFileSync(
+	join( process.env.WP_ARTIFACTS_PATH, '/performance-results.md' ),
+	summaryMarkdown
+);
+
 if ( summaryFile ) {
-	writeFileSync(
-		summaryFile,
-		summaryMarkdown
-	);
+	writeFileSync( summaryFile, summaryMarkdown );
 }
diff --git a/tests/performance/config/global-setup.js b/tests/performance/config/global-setup.js
index f3a0a4f26a691..25e99a47d8457 100644
--- a/tests/performance/config/global-setup.js
+++ b/tests/performance/config/global-setup.js
@@ -30,9 +30,7 @@ async function globalSetup( config ) {
 	await requestUtils.setupRest();
 
 	// Reset the test environment before running the tests.
-	await Promise.all( [
-		requestUtils.activateTheme( 'twentytwentyone' ),
-	] );
+	await Promise.all( [ requestUtils.activateTheme( 'twentytwentyone' ) ] );
 
 	await requestContext.dispose();
 }
diff --git a/tests/performance/config/performance-reporter.js b/tests/performance/config/performance-reporter.js
index a617fcbe343fc..7e6a62f10bfe5 100644
--- a/tests/performance/config/performance-reporter.js
+++ b/tests/performance/config/performance-reporter.js
@@ -1,19 +1,23 @@
 /**
  * External dependencies
  */
-import { join, basename } from 'node:path';
-import { writeFileSync } from 'node:fs';
-
-/**
- * Internal dependencies
- */
-import { getResultsFilename } from '../utils';
+import { join } from 'node:path';
+import { writeFileSync, existsSync, mkdirSync } from 'node:fs';
 
 /**
  * @implements {import('@playwright/test/reporter').Reporter}
  */
 class PerformanceReporter {
 	/**
+	 *
+	 * @type {Record<string,{title: string; results: Record< string, number[] >[];}>}
+	 */
+	allResults = {};
+
+	/**
+	 * Called after a test has been finished in the worker process.
+	 *
+	 * Used to add test results to the final summary of all tests.
 	 *
 	 * @param {import('@playwright/test/reporter').TestCase} test
 	 * @param {import('@playwright/test/reporter').TestResult} result
@@ -24,15 +28,53 @@ class PerformanceReporter {
 		);
 
 		if ( performanceResults?.body ) {
-			writeFileSync(
-				join(
-					process.env.WP_ARTIFACTS_PATH,
-					getResultsFilename( basename( test.location.file, '.test.js' ) )
-				),
-				performanceResults.body.toString( 'utf-8' )
+			this.allResults[ test.location.file ] ??= {
+				// 0 = empty, 1 = browser, 2 = file name, 3 = test suite name.
+				title: test.titlePath()[ 3 ],
+				results: [],
+			};
+			this.allResults[ test.location.file ].results.push(
+				JSON.parse( performanceResults.body.toString( 'utf-8' ) )
 			);
 		}
 	}
+
+	/**
+	 * Called after all tests have been run, or testing has been interrupted.
+	 *
+	 * Writes all raw numbers to a file for further processing,
+	 * for example to compare with a previous run.
+	 *
+	 * @param {import('@playwright/test/reporter').FullResult} result
+	 */
+	onEnd( result ) {
+		const summary = [];
+
+		for ( const [ file, { title, results } ] of Object.entries(
+			this.allResults
+		) ) {
+			summary.push( {
+				file,
+				title,
+				results,
+			} );
+		}
+
+		if ( ! existsSync( process.env.WP_ARTIFACTS_PATH ) ) {
+			mkdirSync( process.env.WP_ARTIFACTS_PATH );
+		}
+
+		const prefix = process.env.TEST_RESULTS_PREFIX;
+		const fileNamePrefix = prefix ? `${ prefix }-` : '';
+
+		writeFileSync(
+			join(
+				process.env.WP_ARTIFACTS_PATH,
+				`${ fileNamePrefix }performance-results.json`
+			),
+			JSON.stringify( summary, null, 2 )
+		);
+	}
 }
 
 export default PerformanceReporter;
diff --git a/tests/performance/log-results.js b/tests/performance/log-results.js
index 14c836ff671a4..9ac0fefc9fe4a 100644
--- a/tests/performance/log-results.js
+++ b/tests/performance/log-results.js
@@ -6,7 +6,8 @@
 const fs = require( 'fs' );
 const path = require( 'path' );
 const https = require( 'https' );
-const [ token, branch, hash, baseHash, timestamp, host ] = process.argv.slice( 2 );
+const [ token, branch, hash, baseHash, timestamp, host ] =
+	process.argv.slice( 2 );
 const { median } = require( './utils' );
 
 // The list of test suites to log.
@@ -20,16 +21,16 @@ const testSuites = [
 ];
 
 // A list of results to parse based on test suites.
-const testResults = testSuites.map(( key ) => ({
+const testResults = testSuites.map( ( key ) => ( {
 	key,
 	file: `${ key }.test.results.json`,
-}));
+} ) );
 
 // A list of base results to parse based on test suites.
-const baseResults = testSuites.map(( key ) => ({
+const baseResults = testSuites.map( ( key ) => ( {
 	key,
 	file: `base-${ key }.test.results.json`,
-}));
+} ) );
 
 /**
  * Parse test files into JSON objects.
@@ -37,11 +38,10 @@ const baseResults = testSuites.map(( key ) => ({
  * @param {string} fileName The name of the file.
  * @returns An array of parsed objects from each file.
  */
-const parseFile = ( fileName ) => (
+const parseFile = ( fileName ) =>
 	JSON.parse(
 		fs.readFileSync( path.join( __dirname, '/specs/', fileName ), 'utf8' )
-	)
-);
+	);
 
 /**
  * Gets the array of metrics from a list of results.
@@ -50,22 +50,19 @@ const parseFile = ( fileName ) => (
  * @return {Object[]} Metrics.
  */
 const formatResults = ( results ) => {
-	return results.reduce(
-		( result, { key, file } ) => {
-			return {
-				...result,
-				...Object.fromEntries(
-					Object.entries(
-						parseFile( file ) ?? {}
-					).map( ( [ metric, value ] ) => [
+	return results.reduce( ( result, { key, file } ) => {
+		return {
+			...result,
+			...Object.fromEntries(
+				Object.entries( parseFile( file ) ?? {} ).map(
+					( [ metric, value ] ) => [
 						key + '-' + metric,
-						median ( value ),
-					] )
-				),
-			};
-		},
-		{}
-	);
+						median( value ),
+					]
+				)
+			),
+		};
+	}, {} );
 };
 
 const data = new TextEncoder().encode(
diff --git a/tests/performance/playwright.config.js b/tests/performance/playwright.config.js
index 1d2781f73ca4d..7ef1be81d2ad9 100644
--- a/tests/performance/playwright.config.js
+++ b/tests/performance/playwright.config.js
@@ -37,4 +37,3 @@ const config = defineConfig( {
 } );
 
 export default config;
-
diff --git a/tests/performance/utils.js b/tests/performance/utils.js
index f56380e9c241a..9973c36c0b6de 100644
--- a/tests/performance/utils.js
+++ b/tests/performance/utils.js
@@ -13,27 +13,102 @@ function median( array ) {
 		: ( numbers[ mid - 1 ] + numbers[ mid ] ) / 2;
 }
 
+function camelCaseDashes( str ) {
+	return str.replace( /-([a-z])/g, function ( g ) {
+		return g[ 1 ].toUpperCase();
+	} );
+}
+
 /**
- * Gets the result file name.
+ * Formats an array of objects as a Markdown table.
+ *
+ * For example, this array:
  *
- * @param {string} fileName File name.
+ * [
+ * 	{
+ * 	    foo: 123,
+ * 	    bar: 456,
+ * 	    baz: 'Yes',
+ * 	},
+ * 	{
+ * 	    foo: 777,
+ * 	    bar: 999,
+ * 	    baz: 'No',
+ * 	}
+ * ]
  *
- * @return {string} Result file name.
+ * Will result in the following table:
+ *
+ * | foo | bar | baz |
+ * |-----|-----|-----|
+ * | 123 | 456 | Yes |
+ * | 777 | 999 | No  |
+ *
+ * @param {Array<Object>} rows Table rows.
+ * @returns {string} Markdown table content.
  */
-function getResultsFilename( fileName ) {
-	const prefix = process.env.TEST_RESULTS_PREFIX;
-	const fileNamePrefix = prefix ? `${ prefix }-` : '';
-	return `${fileNamePrefix + fileName}.results.json`;
+function formatAsMarkdownTable( rows ) {
+	let result = '';
+	const headers = Object.keys( rows[ 0 ] );
+	for ( const header of headers ) {
+		result += `| ${ header } `;
+	}
+	result += '|\n';
+	for ( const header of headers ) {
+		result += '| ------ ';
+	}
+	result += '|\n';
+
+	for ( const row of rows ) {
+		for ( const value of Object.values( row ) ) {
+			result += `| ${ value } `;
+		}
+		result += '|\n';
+	}
+
+	return result;
 }
 
-function camelCaseDashes( str ) {
-	return str.replace( /-([a-z])/g, function( g ) {
-		return g[ 1 ].toUpperCase();
-	} );
+/**
+ * Nicely formats a given value.
+ *
+ * @param {string} metric Metric.
+ * @param {number} value
+ */
+function formatValue( metric, value ) {
+	if ( null === value ) {
+		return 'N/A';
+	}
+	if ( 'wpMemoryUsage' === metric ) {
+		return `${ ( value / Math.pow( 10, 6 ) ).toFixed( 2 ) } MB`;
+	}
+
+	return `${ value.toFixed( 2 ) } ms`;
+}
+
+/**
+ * Returns a Markdown link to a Git commit on the current GitHub repository.
+ *
+ * For example, turns `a5c3785ed8d6a35868bc169f07e40e889087fd2e`
+ * into (https://github.com/wordpress/wordpress-develop/commit/36fe58a8c64dcc83fc21bddd5fcf054aef4efb27)[36fe58a].
+ *
+ * @param {string} sha Commit SHA.
+ * @return string Link
+ */
+function linkToSha( sha ) {
+	const repoName =
+		process.env.GITHUB_REPOSITORY || 'wordpress/wordpress-develop';
+
+	return `[${ sha.slice(
+		0,
+		7
+	) }](https://github.com/${ repoName }/commit/${ sha })`;
 }
 
 module.exports = {
 	median,
-	getResultsFilename,
 	camelCaseDashes,
+	formatAsMarkdownTable,
+	formatValue,
+	linkToSha,
 };

From 8095e996954d39ec68a395abbc6d755d0f756e3c Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Sat, 27 Apr 2024 09:55:25 +0200
Subject: [PATCH 14/51] Fix import

---
 tests/performance/compare-results.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/performance/compare-results.js b/tests/performance/compare-results.js
index 2e6af694953fa..ab6951314ba00 100644
--- a/tests/performance/compare-results.js
+++ b/tests/performance/compare-results.js
@@ -3,8 +3,8 @@
 /**
  * External dependencies.
  */
-const { readFileSync, writeFileSync } = require( 'node:fs' );
-const { join, existsSync } = require( 'node:path' );
+const { readFileSync, writeFileSync, existsSync } = require( 'node:fs' );
+const { join } = require( 'node:path' );
 
 /**
  * Internal dependencies

From b80659c51843cd02c6b00f2db08c6021c79f0aaa Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Sat, 27 Apr 2024 11:05:05 +0200
Subject: [PATCH 15/51] Upload artifacts for debugging

---
 .github/workflows/performance.yml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index 46303b3a46e31..862dc69c6c1b9 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -245,6 +245,17 @@ jobs:
           TEST_RESULTS_PREFIX: base
         run: npm run test:performance
 
+      - name: Archive artifacts
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        if: always()
+        with:
+          name: performance-artifacts-${{ github.run_id }}
+          path: artifacts
+          if-no-files-found: ignore
+
+      - name: Ensure version-controlled files are not modified or deleted
+        run: git diff --exit-code
+
       - name: Compare results with base
         run: node ./tests/performance/compare-results.js ${{ runner.temp }}/summary.md
 

From 2b328d8bdcc18bfc808edc32d8ac4e8cd63ce68a Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Sat, 27 Apr 2024 11:12:23 +0200
Subject: [PATCH 16/51] Fix results structure

---
 tests/performance/compare-results.js             |  4 ++--
 tests/performance/config/performance-reporter.js | 14 ++++++++++----
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/tests/performance/compare-results.js b/tests/performance/compare-results.js
index ab6951314ba00..b46e53a8342af 100644
--- a/tests/performance/compare-results.js
+++ b/tests/performance/compare-results.js
@@ -70,8 +70,8 @@ console.log(
 	'Note: Due to the nature of how GitHub Actions work, some variance in the results is expected.\n'
 );
 
-for ( const { file, title, results } of afterStats ) {
-	const prevStat = beforeStats.find( ( s ) => s.file === file );
+for ( const { title, results } of afterStats ) {
+	const prevStat = beforeStats.find( ( s ) => s.title === title );
 
 	/**
 	 * @type {Array<Record<string, string>>}
diff --git a/tests/performance/config/performance-reporter.js b/tests/performance/config/performance-reporter.js
index 7e6a62f10bfe5..4b0e40a0de769 100644
--- a/tests/performance/config/performance-reporter.js
+++ b/tests/performance/config/performance-reporter.js
@@ -28,11 +28,17 @@ class PerformanceReporter {
 		);
 
 		if ( performanceResults?.body ) {
-			this.allResults[ test.location.file ] ??= {
-				// 0 = empty, 1 = browser, 2 = file name, 3 = test suite name.
-				title: test.titlePath()[ 3 ],
+			// 0 = empty, 1 = browser, 2 = file name, 3 = test suite name, 4 = test name.
+			const titlePath = test.titlePath();
+			const title = `${ titlePath[ 3 ] } › ${ titlePath[ 4 ] }`;
+
+			// results is an array in case repeatEach is > 1.
+
+			this.allResults[ title ] ??= {
+				file: test.location.file, // Unused, but useful for debugging.
 				results: [],
 			};
+
 			this.allResults[ test.location.file ].results.push(
 				JSON.parse( performanceResults.body.toString( 'utf-8' ) )
 			);
@@ -50,7 +56,7 @@ class PerformanceReporter {
 	onEnd( result ) {
 		const summary = [];
 
-		for ( const [ file, { title, results } ] of Object.entries(
+		for ( const [ title, { file, results } ] of Object.entries(
 			this.allResults
 		) ) {
 			summary.push( {

From b1e7826f34664ce681929134180cbcdd081164b5 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Sat, 27 Apr 2024 11:25:14 +0200
Subject: [PATCH 17/51] Fix artifact name

---
 .github/workflows/performance.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index 862dc69c6c1b9..f94ed79f8e49a 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -249,7 +249,7 @@ jobs:
         uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         if: always()
         with:
-          name: performance-artifacts-${{ github.run_id }}
+          name: performance-artifacts${{ matrix.memcached && '-memcached' || '' }}-${{ github.run_id }}
           path: artifacts
           if-no-files-found: ignore
 

From 0bcac093c6e23b6f878da122257a64d5d2e7f501 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Sat, 27 Apr 2024 13:09:15 +0200
Subject: [PATCH 18/51] Use correct var

---
 tests/performance/config/performance-reporter.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/performance/config/performance-reporter.js b/tests/performance/config/performance-reporter.js
index 4b0e40a0de769..9617625eedc7c 100644
--- a/tests/performance/config/performance-reporter.js
+++ b/tests/performance/config/performance-reporter.js
@@ -39,7 +39,7 @@ class PerformanceReporter {
 				results: [],
 			};
 
-			this.allResults[ test.location.file ].results.push(
+			this.allResults[ title ].results.push(
 				JSON.parse( performanceResults.body.toString( 'utf-8' ) )
 			);
 		}

From 2e8fc1c9566c7126c7619d1f86ad1b9d82093bca Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Sat, 27 Apr 2024 13:10:50 +0200
Subject: [PATCH 19/51] Hardening

---
 tests/performance/compare-results.js | 6 +++++-
 tests/performance/utils.js           | 5 +++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/tests/performance/compare-results.js b/tests/performance/compare-results.js
index b46e53a8342af..b9db20e88da6f 100644
--- a/tests/performance/compare-results.js
+++ b/tests/performance/compare-results.js
@@ -104,7 +104,11 @@ for ( const { title, results } of afterStats ) {
 	}
 
 	console.log( title );
-	console.table( rows );
+	if ( rows.length > 0 ) {
+		console.table( rows );
+	} else {
+		console.log( '(no results)' );
+	}
 
 	summaryMarkdown += `**${ title }**\n\n`;
 	summaryMarkdown += `${ formatAsMarkdownTable( rows ) }\n`;
diff --git a/tests/performance/utils.js b/tests/performance/utils.js
index 9973c36c0b6de..a392bfad2d731 100644
--- a/tests/performance/utils.js
+++ b/tests/performance/utils.js
@@ -49,6 +49,11 @@ function camelCaseDashes( str ) {
  */
 function formatAsMarkdownTable( rows ) {
 	let result = '';
+
+	if ( ! rows.length ) {
+		return result;
+	}
+
 	const headers = Object.keys( rows[ 0 ] );
 	for ( const header of headers ) {
 		result += `| ${ header } `;

From 1bcace72ce3745a45f7e2707f14155744fe55cbc Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Sun, 28 Apr 2024 12:41:16 +0200
Subject: [PATCH 20/51] Get previous build from artifact

---
 .github/workflows/performance.yml | 44 ++++++++++++-------------------
 1 file changed, 17 insertions(+), 27 deletions(-)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index f94ed79f8e49a..9083669f95762 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -186,26 +186,27 @@ jobs:
       - name: Run performance tests (current commit)
         run: npm run test:performance
 
-      - name: Check out target commit (target branch or previous commit)
+      - name: Get previous build (target branch or previous commit)
+        id: get-previous-build
         run: |
-          if [[ -z "$TARGET_REF" ]]; then
-            git fetch -n origin $TARGET_SHA
-          else
-            git fetch -n origin $TARGET_REF
-          fi
-          git reset --hard $TARGET_SHA
+          echo "artifact_id=$(gh api repos/WordPress/wordpress-develop/actions/artifacts?name='wordpress-build-$TARGET_SHA' --jq ".artifacts[] .id")" >> $GITHUB_OUTPUT
 
-      - name: Set up Node.js
-        uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
+      - name: Download artifact
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         with:
-          node-version-file: '.nvmrc'
-          cache: npm
+          script: |
+            const download = await github.rest.actions.downloadArtifact( {
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: ${{ steps.get-previous-build.outputs.artifact_id }}
+               archive_format: 'zip',
+            } );
 
-      - name: Install npm dependencies
-        run: npm ci
+            const fs = require( 'fs' );
+            fs.writeFileSync( '${{ env.LOCAL_DIR }}/before.zip', Buffer.from( download.data ) )
 
-      - name: Build WordPress
-        run: npm run build
+      - name: Unzip the build
+        run: unzip ${{ env.LOCAL_DIR }}/before.zip
 
       - name: Flush cache
         run: npm run env:cli -- cache flush --path=/var/www/${{ env.LOCAL_DIR }}
@@ -218,18 +219,6 @@ jobs:
           TEST_RESULTS_PREFIX: before
         run: npm run test:performance
 
-      - name: Reset to original commit
-        run: git reset --hard $GITHUB_SHA
-
-      - name: Set up Node.js
-        uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
-        with:
-          node-version-file: '.nvmrc'
-          cache: npm
-
-      - name: Install npm dependencies
-        run: npm ci
-
       - name: Set the environment to the baseline version
         run: |
           npm run env:cli -- core update --version=${{ env.BASE_TAG }} --force --path=/var/www/${{ env.LOCAL_DIR }}
@@ -237,6 +226,7 @@ jobs:
 
       - name: Flush cache
         run: npm run env:cli -- cache flush --path=/var/www/${{ env.LOCAL_DIR }}
+
       - name: Run any database upgrades
         run: npm run env:cli -- core update-db --path=/var/www/${{ env.LOCAL_DIR }}
 

From b4585b81ecc0f0ba791c6dcb7efab9f912264069 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Sun, 28 Apr 2024 13:00:40 +0200
Subject: [PATCH 21/51] Pass gh token

---
 .github/workflows/performance.yml           | 2 ++
 .github/workflows/pull-request-comments.yml | 1 +
 2 files changed, 3 insertions(+)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index 9083669f95762..691cb9fb2c139 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -190,6 +190,8 @@ jobs:
         id: get-previous-build
         run: |
           echo "artifact_id=$(gh api repos/WordPress/wordpress-develop/actions/artifacts?name='wordpress-build-$TARGET_SHA' --jq ".artifacts[] .id")" >> $GITHUB_OUTPUT
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Download artifact
         uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
diff --git a/.github/workflows/pull-request-comments.yml b/.github/workflows/pull-request-comments.yml
index df276270f1396..34651c9da7588 100644
--- a/.github/workflows/pull-request-comments.yml
+++ b/.github/workflows/pull-request-comments.yml
@@ -92,6 +92,7 @@ jobs:
       - name: Download artifact
         uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
             const artifacts = await github.rest.actions.listWorkflowRunArtifacts( {
                owner: context.repo.owner,

From a86db9f37c8b80806134d4c26da6f55dd593adea Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Sun, 28 Apr 2024 14:05:07 +0200
Subject: [PATCH 22/51] Double quotes, add missing comma

---
 .github/workflows/performance.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index 691cb9fb2c139..ccb4ff73d7bc6 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -189,7 +189,7 @@ jobs:
       - name: Get previous build (target branch or previous commit)
         id: get-previous-build
         run: |
-          echo "artifact_id=$(gh api repos/WordPress/wordpress-develop/actions/artifacts?name='wordpress-build-$TARGET_SHA' --jq ".artifacts[] .id")" >> $GITHUB_OUTPUT
+          echo "artifact_id=$(gh api repos/WordPress/wordpress-develop/actions/artifacts?name="wordpress-build-$TARGET_SHA" --jq ".artifacts[] .id")" >> $GITHUB_OUTPUT
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -200,7 +200,7 @@ jobs:
             const download = await github.rest.actions.downloadArtifact( {
                owner: context.repo.owner,
                repo: context.repo.repo,
-               artifact_id: ${{ steps.get-previous-build.outputs.artifact_id }}
+               artifact_id: ${{ steps.get-previous-build.outputs.artifact_id || null }},
                archive_format: 'zip',
             } );
 

From fb4cfeb9c679090c5345772ab2341ed319e5590e Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Sun, 28 Apr 2024 14:16:10 +0200
Subject: [PATCH 23/51] Fix unzip step

---
 .github/workflows/performance.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index ccb4ff73d7bc6..638062720ad55 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -205,10 +205,13 @@ jobs:
             } );
 
             const fs = require( 'fs' );
-            fs.writeFileSync( '${{ env.LOCAL_DIR }}/before.zip', Buffer.from( download.data ) )
+            fs.writeFileSync( '${{ github.workspace }}/before.zip', Buffer.from( download.data ) )
 
       - name: Unzip the build
-        run: unzip ${{ env.LOCAL_DIR }}/before.zip
+        run: |
+          rm -rf ${{ env.LOCAL_DIR }}
+          unzip ${{ github.workspace }}/before.zip
+          unzip ${{ github.workspace }}/wordpress.zip
 
       - name: Flush cache
         run: npm run env:cli -- cache flush --path=/var/www/${{ env.LOCAL_DIR }}

From 3021835ace9dedded8b325e8eab954c7363c2102 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Sun, 28 Apr 2024 14:18:39 +0200
Subject: [PATCH 24/51] Fix comparison

---
 tests/performance/compare-results.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/performance/compare-results.js b/tests/performance/compare-results.js
index b9db20e88da6f..06b0a302c68f6 100644
--- a/tests/performance/compare-results.js
+++ b/tests/performance/compare-results.js
@@ -85,7 +85,7 @@ for ( const { title, results } of afterStats ) {
 			// Only do comparison if the number of results is the same.
 			const prevValues =
 				prevStat?.results.length === results.length
-					? prevStat?.results[ i ].key
+					? prevStat?.results[ i ][ metric ]
 					: null;
 
 			const value = median( values );

From 1cd06a8cc40ee3e176223775326485d4c360eb2e Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Sun, 28 Apr 2024 15:17:11 +0200
Subject: [PATCH 25/51] Unzip with override instead

---
 .github/workflows/performance.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index 638062720ad55..adf8aeb5adf56 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -209,9 +209,8 @@ jobs:
 
       - name: Unzip the build
         run: |
-          rm -rf ${{ env.LOCAL_DIR }}
           unzip ${{ github.workspace }}/before.zip
-          unzip ${{ github.workspace }}/wordpress.zip
+          unzip -o ${{ github.workspace }}/wordpress.zip
 
       - name: Flush cache
         run: npm run env:cli -- cache flush --path=/var/www/${{ env.LOCAL_DIR }}

From cde781ea9620c5f687be6eec786b3c6365041492 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Sun, 28 Apr 2024 15:19:50 +0200
Subject: [PATCH 26/51] Track number of db queries

---
 tests/performance/wp-content/mu-plugins/server-timing.php | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/performance/wp-content/mu-plugins/server-timing.php b/tests/performance/wp-content/mu-plugins/server-timing.php
index 53f83fea796c3..c8f8b7c592a9f 100644
--- a/tests/performance/wp-content/mu-plugins/server-timing.php
+++ b/tests/performance/wp-content/mu-plugins/server-timing.php
@@ -31,6 +31,7 @@ static function () use ( $server_timing_values, $template_start ) {
 				 * This is a nice little trick as it allows to easily get this information in JS.
 				 */
 				$server_timing_values['memory-usage'] = memory_get_usage();
+				$server_timing_values['db-queries']   = $GLOBALS['wpdb']->num_queries;
 
 				$header_values = array();
 				foreach ( $server_timing_values as $slug => $value ) {

From 70eba934f3fd6857283143fd509c4983091c9b7b Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Sun, 28 Apr 2024 15:20:41 +0200
Subject: [PATCH 27/51] Track obj cache usage

---
 tests/performance/wp-content/mu-plugins/server-timing.php | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/performance/wp-content/mu-plugins/server-timing.php b/tests/performance/wp-content/mu-plugins/server-timing.php
index c8f8b7c592a9f..e50d3ebc06c82 100644
--- a/tests/performance/wp-content/mu-plugins/server-timing.php
+++ b/tests/performance/wp-content/mu-plugins/server-timing.php
@@ -30,8 +30,9 @@ static function () use ( $server_timing_values, $template_start ) {
 				 * any numeric value can actually be passed.
 				 * This is a nice little trick as it allows to easily get this information in JS.
 				 */
-				$server_timing_values['memory-usage'] = memory_get_usage();
-				$server_timing_values['db-queries']   = $GLOBALS['wpdb']->num_queries;
+				$server_timing_values['memory-usage']  = memory_get_usage();
+				$server_timing_values['db-queries']    = $GLOBALS['wpdb']->num_queries;
+				$server_timing_values['ext-obj-cache'] = wp_using_ext_object_cache() ? 1 : 0;
 
 				$header_values = array();
 				foreach ( $server_timing_values as $slug => $value ) {

From 44aff66b3d60fd248cc3cdc3c1516a5991f781f1 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Sun, 28 Apr 2024 15:48:44 +0200
Subject: [PATCH 28/51] Fix query count

---
 .../performance/wp-content/mu-plugins/server-timing.php  | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/tests/performance/wp-content/mu-plugins/server-timing.php b/tests/performance/wp-content/mu-plugins/server-timing.php
index e50d3ebc06c82..8c57f31f03314 100644
--- a/tests/performance/wp-content/mu-plugins/server-timing.php
+++ b/tests/performance/wp-content/mu-plugins/server-timing.php
@@ -4,7 +4,7 @@
 	'template_include',
 	static function ( $template ) {
 
-		global $timestart;
+		global $timestart, $wpdb;
 
 		$server_timing_values = array();
 		$template_start       = microtime( true );
@@ -15,10 +15,7 @@ static function ( $template ) {
 
 		add_action(
 			'shutdown',
-			static function () use ( $server_timing_values, $template_start ) {
-
-				global $timestart;
-
+			static function () use ( $server_timing_values, $template_start, $wpdb ) {
 				$output = ob_get_clean();
 
 				$server_timing_values['template'] = microtime( true ) - $template_start;
@@ -31,7 +28,7 @@ static function () use ( $server_timing_values, $template_start ) {
 				 * This is a nice little trick as it allows to easily get this information in JS.
 				 */
 				$server_timing_values['memory-usage']  = memory_get_usage();
-				$server_timing_values['db-queries']    = $GLOBALS['wpdb']->num_queries;
+				$server_timing_values['db-queries']    = $wpdb->num_queries;
 				$server_timing_values['ext-obj-cache'] = wp_using_ext_object_cache() ? 1 : 0;
 
 				$header_values = array();

From 72456a53dbb6aabf1fec7f87ffb0753e82a9e73e Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Sun, 28 Apr 2024 15:48:54 +0200
Subject: [PATCH 29/51] Improve formatting

---
 tests/performance/compare-results.js | 5 +++--
 tests/performance/utils.js           | 5 +++++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/tests/performance/compare-results.js b/tests/performance/compare-results.js
index 06b0a302c68f6..89132e9ff99cd 100644
--- a/tests/performance/compare-results.js
+++ b/tests/performance/compare-results.js
@@ -92,13 +92,14 @@ for ( const { title, results } of afterStats ) {
 			const prevValue = prevValues ? median( prevValues ) : 0;
 			const delta = value - prevValue;
 			const percentage = ( delta / value ) * 100;
+			const showDiff = metric !== 'wpExtObjCache';
 
 			rows.push( {
 				Metric: metric,
 				Before: formatValue( metric, prevValue ),
 				After: formatValue( metric, value ),
-				'Diff abs.': formatValue( metric, delta ),
-				'Diff %': `${ percentage.toFixed( 2 ) } %`,
+				'Diff abs.': showDiff ? formatValue( metric, delta ) : '',
+				'Diff %': showDiff ? `${ percentage.toFixed( 2 ) } %` : '',
 			} );
 		}
 	}
diff --git a/tests/performance/utils.js b/tests/performance/utils.js
index a392bfad2d731..ba5a74099afde 100644
--- a/tests/performance/utils.js
+++ b/tests/performance/utils.js
@@ -84,10 +84,15 @@ function formatValue( metric, value ) {
 	if ( null === value ) {
 		return 'N/A';
 	}
+
 	if ( 'wpMemoryUsage' === metric ) {
 		return `${ ( value / Math.pow( 10, 6 ) ).toFixed( 2 ) } MB`;
 	}
 
+	if ( 'wpExtObjCache' === metric || 'wpDbQueries' === metric ) {
+		return value;
+	}
+
 	return `${ value.toFixed( 2 ) } ms`;
 }
 

From a49f3cd31af4b10775968b5a371f0bc1a3923d25 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Sun, 28 Apr 2024 15:50:55 +0200
Subject: [PATCH 30/51] NaN check

---
 tests/performance/compare-results.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/performance/compare-results.js b/tests/performance/compare-results.js
index 89132e9ff99cd..71ba026938361 100644
--- a/tests/performance/compare-results.js
+++ b/tests/performance/compare-results.js
@@ -92,7 +92,7 @@ for ( const { title, results } of afterStats ) {
 			const prevValue = prevValues ? median( prevValues ) : 0;
 			const delta = value - prevValue;
 			const percentage = ( delta / value ) * 100;
-			const showDiff = metric !== 'wpExtObjCache';
+			const showDiff = metric !== 'wpExtObjCache' && ! Number.isNaN( percentage );
 
 			rows.push( {
 				Metric: metric,

From 1b4ffabad41747fcb39c725e119d218d55c6a18f Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Sun, 28 Apr 2024 15:59:18 +0200
Subject: [PATCH 31/51] Only run baseline on trunk

---
 .github/workflows/performance.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index adf8aeb5adf56..fb1e1fe36c5e8 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -224,17 +224,21 @@ jobs:
         run: npm run test:performance
 
       - name: Set the environment to the baseline version
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
         run: |
           npm run env:cli -- core update --version=${{ env.BASE_TAG }} --force --path=/var/www/${{ env.LOCAL_DIR }}
           npm run env:cli -- core version --path=/var/www/${{ env.LOCAL_DIR }}
 
       - name: Flush cache
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
         run: npm run env:cli -- cache flush --path=/var/www/${{ env.LOCAL_DIR }}
 
       - name: Run any database upgrades
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
         run: npm run env:cli -- core update-db --path=/var/www/${{ env.LOCAL_DIR }}
 
       - name: Run baseline performance tests
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
         env:
           TEST_RESULTS_PREFIX: base
         run: npm run test:performance

From b224b22a223cc0c8c4aad4ad88203ba290349b47 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Sun, 28 Apr 2024 23:12:03 +0200
Subject: [PATCH 32/51] Further tests

---
 .github/workflows/performance.yml      | 15 +++++++++++++--
 tests/performance/compare-results.js   | 18 +++++++++++++++---
 tests/performance/playwright.config.js |  1 +
 tests/performance/utils.js             | 20 ++++++++++++++++++++
 4 files changed, 49 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index fb1e1fe36c5e8..a56cc200635c7 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -168,9 +168,11 @@ jobs:
           npm run env:cli -- import themeunittestdata.wordpress.xml --authors=create --path=/var/www/${{ env.LOCAL_DIR }}
           rm themeunittestdata.wordpress.xml
 
+      - name: Deactivate WordPress Importer plugin
+        run: npm run env:cli -- plugin deactivate wordpress-importer --path=/var/www/${{ env.LOCAL_DIR }}
+
       - name: Update permalink structure
-        run: |
-          npm run env:cli -- rewrite structure '/%year%/%monthnum%/%postname%/' --path=/var/www/${{ env.LOCAL_DIR }}
+        run: npm run env:cli -- rewrite structure '/%year%/%monthnum%/%postname%/' --path=/var/www/${{ env.LOCAL_DIR }}
 
       - name: Install additional languages
         run: |
@@ -178,6 +180,15 @@ jobs:
           npm run env:cli -- language plugin install de_DE --all --path=/var/www/${{ env.LOCAL_DIR }}
           npm run env:cli -- language theme install de_DE --all --path=/var/www/${{ env.LOCAL_DIR }}
 
+      - name: Disable external HTTP requests
+        run: npm run env:cli -- config set WP_HTTP_BLOCK_EXTERNAL true --raw --type=constant --path=/var/www/${{ env.LOCAL_DIR }}
+
+      - name: Disable cron
+        run: npm run env:cli -- config set DISABLE_WP_CRON true --raw --type=constant --path=/var/www/${{ env.LOCAL_DIR }}
+
+      - name: List defined constants
+        run: npm run env:cli -- config list --path=/var/www/${{ env.LOCAL_DIR }}
+
       - name: Install MU plugin
         run: |
           mkdir ./${{ env.LOCAL_DIR }}/wp-content/mu-plugins
diff --git a/tests/performance/compare-results.js b/tests/performance/compare-results.js
index 71ba026938361..30678f5085ab0 100644
--- a/tests/performance/compare-results.js
+++ b/tests/performance/compare-results.js
@@ -14,6 +14,8 @@ const {
 	formatAsMarkdownTable,
 	formatValue,
 	linkToSha,
+	standardDeviation,
+	medianAbsoluteDeviation,
 } = require( './utils' );
 
 process.env.WP_ARTIFACTS_PATH ??= join( process.cwd(), 'artifacts' );
@@ -60,15 +62,23 @@ if ( process.env.TARGET_SHA ) {
 	}
 }
 
+const numberOfIterations = Object.values( afterStats[ 0 ].results[ 0 ] )[ 0 ].length;
+
+summaryMarkdown += `All numbers are median values over ${ numberOfIterations } iterations.\n\n`;
+
 if ( process.env.GITHUB_SHA ) {
 	summaryMarkdown += `**Note:** Due to the nature of how GitHub Actions work, some variance in the results is expected.\n\n`;
 }
 
 console.log( 'Performance Test Results\n' );
 
-console.log(
-	'Note: Due to the nature of how GitHub Actions work, some variance in the results is expected.\n'
-);
+console.log( `All numbers are median values over ${ numberOfIterations } iterations.\n` );
+
+if ( process.env.GITHUB_SHA ) {
+	console.log(
+		'Note: Due to the nature of how GitHub Actions work, some variance in the results is expected.\n'
+	);
+}
 
 for ( const { title, results } of afterStats ) {
 	const prevStat = beforeStats.find( ( s ) => s.title === title );
@@ -100,6 +110,8 @@ for ( const { title, results } of afterStats ) {
 				After: formatValue( metric, value ),
 				'Diff abs.': showDiff ? formatValue( metric, delta ) : '',
 				'Diff %': showDiff ? `${ percentage.toFixed( 2 ) } %` : '',
+				'STD':  showDiff ? formatValue( metric, standardDeviation( values ) ) : '',
+				'MAD':  showDiff ? formatValue( metric, medianAbsoluteDeviation( values ) ) : '',
 			} );
 		}
 	}
diff --git a/tests/performance/playwright.config.js b/tests/performance/playwright.config.js
index 7ef1be81d2ad9..c8d18d1e00e5b 100644
--- a/tests/performance/playwright.config.js
+++ b/tests/performance/playwright.config.js
@@ -23,6 +23,7 @@ const config = defineConfig( {
 	forbidOnly: !! process.env.CI,
 	workers: 1,
 	retries: 0,
+	repeatEach: 2,
 	timeout: parseInt( process.env.TIMEOUT || '', 10 ) || 600_000, // Defaults to 10 minutes.
 	// Don't report slow test "files", as we will be running our tests in serial.
 	reportSlowTests: null,
diff --git a/tests/performance/utils.js b/tests/performance/utils.js
index ba5a74099afde..805d61b2bd557 100644
--- a/tests/performance/utils.js
+++ b/tests/performance/utils.js
@@ -115,10 +115,30 @@ function linkToSha( sha ) {
 	) }](https://github.com/${ repoName }/commit/${ sha })`;
 }
 
+function standardDeviation( array = [] ) {
+	if ( ! array.length ) {
+		return 0;
+	}
+
+	const mean = array.reduce( ( a, b ) => a + b ) / array.length
+	return Math.sqrt( array.map( ( x ) => Math.pow( x - mean, 2 ) ).reduce( ( a, b ) => a + b ) / array.length )
+}
+
+function medianAbsoluteDeviation( array = [] ) {
+	if ( ! array.length ) {
+		return 0;
+	}
+
+	const med = median( array );
+	return median( array.map( ( a ) => Math.abs( a - med ) ) );
+}
+
 module.exports = {
 	median,
 	camelCaseDashes,
 	formatAsMarkdownTable,
 	formatValue,
 	linkToSha,
+	standardDeviation,
+	medianAbsoluteDeviation,
 };

From 645c598b5c8780065122ae42d86bfeccbc0adef1 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Sun, 28 Apr 2024 23:13:14 +0200
Subject: [PATCH 33/51] Formatting

---
 tests/performance/compare-results.js | 18 +++++++++++++-----
 tests/performance/utils.js           |  8 ++++++--
 2 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/tests/performance/compare-results.js b/tests/performance/compare-results.js
index 30678f5085ab0..334066a83cedc 100644
--- a/tests/performance/compare-results.js
+++ b/tests/performance/compare-results.js
@@ -62,7 +62,8 @@ if ( process.env.TARGET_SHA ) {
 	}
 }
 
-const numberOfIterations = Object.values( afterStats[ 0 ].results[ 0 ] )[ 0 ].length;
+const numberOfIterations = Object.values( afterStats[ 0 ].results[ 0 ] )[ 0 ]
+	.length;
 
 summaryMarkdown += `All numbers are median values over ${ numberOfIterations } iterations.\n\n`;
 
@@ -72,7 +73,9 @@ if ( process.env.GITHUB_SHA ) {
 
 console.log( 'Performance Test Results\n' );
 
-console.log( `All numbers are median values over ${ numberOfIterations } iterations.\n` );
+console.log(
+	`All numbers are median values over ${ numberOfIterations } iterations.\n`
+);
 
 if ( process.env.GITHUB_SHA ) {
 	console.log(
@@ -102,7 +105,8 @@ for ( const { title, results } of afterStats ) {
 			const prevValue = prevValues ? median( prevValues ) : 0;
 			const delta = value - prevValue;
 			const percentage = ( delta / value ) * 100;
-			const showDiff = metric !== 'wpExtObjCache' && ! Number.isNaN( percentage );
+			const showDiff =
+				metric !== 'wpExtObjCache' && ! Number.isNaN( percentage );
 
 			rows.push( {
 				Metric: metric,
@@ -110,8 +114,12 @@ for ( const { title, results } of afterStats ) {
 				After: formatValue( metric, value ),
 				'Diff abs.': showDiff ? formatValue( metric, delta ) : '',
 				'Diff %': showDiff ? `${ percentage.toFixed( 2 ) } %` : '',
-				'STD':  showDiff ? formatValue( metric, standardDeviation( values ) ) : '',
-				'MAD':  showDiff ? formatValue( metric, medianAbsoluteDeviation( values ) ) : '',
+				STD: showDiff
+					? formatValue( metric, standardDeviation( values ) )
+					: '',
+				MAD: showDiff
+					? formatValue( metric, medianAbsoluteDeviation( values ) )
+					: '',
 			} );
 		}
 	}
diff --git a/tests/performance/utils.js b/tests/performance/utils.js
index 805d61b2bd557..7c24cd4e06209 100644
--- a/tests/performance/utils.js
+++ b/tests/performance/utils.js
@@ -120,8 +120,12 @@ function standardDeviation( array = [] ) {
 		return 0;
 	}
 
-	const mean = array.reduce( ( a, b ) => a + b ) / array.length
-	return Math.sqrt( array.map( ( x ) => Math.pow( x - mean, 2 ) ).reduce( ( a, b ) => a + b ) / array.length )
+	const mean = array.reduce( ( a, b ) => a + b ) / array.length;
+	return Math.sqrt(
+		array
+			.map( ( x ) => Math.pow( x - mean, 2 ) )
+			.reduce( ( a, b ) => a + b ) / array.length
+	);
 }
 
 function medianAbsoluteDeviation( array = [] ) {

From 3522a0185a185b59a3ceb5e5b1458f912276bee0 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Mon, 29 Apr 2024 09:50:11 +0200
Subject: [PATCH 34/51] Explain number of reps

---
 tests/performance/compare-results.js | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/performance/compare-results.js b/tests/performance/compare-results.js
index 334066a83cedc..0833a3d1e3353 100644
--- a/tests/performance/compare-results.js
+++ b/tests/performance/compare-results.js
@@ -62,10 +62,14 @@ if ( process.env.TARGET_SHA ) {
 	}
 }
 
+const numberOfRepetitions = afterStats[ 0 ].results.length;
 const numberOfIterations = Object.values( afterStats[ 0 ].results[ 0 ] )[ 0 ]
 	.length;
 
-summaryMarkdown += `All numbers are median values over ${ numberOfIterations } iterations.\n\n`;
+const repetitions = `${ numberOfRepetitions } ${numberOfRepetitions === 1 ? 'repetition' : 'repetitions'}`;
+const iterations  = `${ numberOfIterations} ${numberOfIterations === 1 ? 'iteration' : 'iterations'}`;
+
+summaryMarkdown += `All numbers are median values over ${ repetitions } with ${ iterations } each.\n\n`;
 
 if ( process.env.GITHUB_SHA ) {
 	summaryMarkdown += `**Note:** Due to the nature of how GitHub Actions work, some variance in the results is expected.\n\n`;
@@ -74,7 +78,7 @@ if ( process.env.GITHUB_SHA ) {
 console.log( 'Performance Test Results\n' );
 
 console.log(
-	`All numbers are median values over ${ numberOfIterations } iterations.\n`
+	`All numbers are median values over ${ repetitions } with ${ iterations } each.\n`
 );
 
 if ( process.env.GITHUB_SHA ) {

From b00715cff34a5296b1bb122957c4f5a629319282 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Mon, 29 Apr 2024 10:14:02 +0200
Subject: [PATCH 35/51] Formatting

---
 tests/performance/utils.js | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/performance/utils.js b/tests/performance/utils.js
index 7c24cd4e06209..9bf6eaa1b0c62 100644
--- a/tests/performance/utils.js
+++ b/tests/performance/utils.js
@@ -89,7 +89,11 @@ function formatValue( metric, value ) {
 		return `${ ( value / Math.pow( 10, 6 ) ).toFixed( 2 ) } MB`;
 	}
 
-	if ( 'wpExtObjCache' === metric || 'wpDbQueries' === metric ) {
+	if ( 'wpExtObjCache' === metric ) {
+		return 1 === value ? 'yes' : 'no';
+	}
+
+	if ( 'wpDbQueries' === metric ) {
 		return value;
 	}
 

From 2e7f4f320d05b7528ca93ff81e378225141e9703 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Mon, 29 Apr 2024 10:14:33 +0200
Subject: [PATCH 36/51] Acc results

---
 tests/performance/compare-results.js | 90 ++++++++++++++++++----------
 1 file changed, 57 insertions(+), 33 deletions(-)

diff --git a/tests/performance/compare-results.js b/tests/performance/compare-results.js
index 0833a3d1e3353..0b8ca628df6e7 100644
--- a/tests/performance/compare-results.js
+++ b/tests/performance/compare-results.js
@@ -66,8 +66,12 @@ const numberOfRepetitions = afterStats[ 0 ].results.length;
 const numberOfIterations = Object.values( afterStats[ 0 ].results[ 0 ] )[ 0 ]
 	.length;
 
-const repetitions = `${ numberOfRepetitions } ${numberOfRepetitions === 1 ? 'repetition' : 'repetitions'}`;
-const iterations  = `${ numberOfIterations} ${numberOfIterations === 1 ? 'iteration' : 'iterations'}`;
+const repetitions = `${ numberOfRepetitions } ${
+	numberOfRepetitions === 1 ? 'repetition' : 'repetitions'
+}`;
+const iterations = `${ numberOfIterations } ${
+	numberOfIterations === 1 ? 'iteration' : 'iterations'
+}`;
 
 summaryMarkdown += `All numbers are median values over ${ repetitions } with ${ iterations } each.\n\n`;
 
@@ -87,6 +91,21 @@ if ( process.env.GITHUB_SHA ) {
 	);
 }
 
+/**
+ *
+ * @param {Array<Record<string, number[]>>} results
+ * @returns {Record<string, number[]>}
+ */
+function accumulateValues( results ) {
+	return results.reduce( ( acc, result ) => {
+		for ( const [ metric, values ] of Object.entries( result ) ) {
+			acc[ metric ] = acc[ metric ] ?? [];
+			acc[ metric ].push( ...values );
+		}
+		return acc;
+	}, {} );
+}
+
 for ( const { title, results } of afterStats ) {
 	const prevStat = beforeStats.find( ( s ) => s.title === title );
 
@@ -95,37 +114,42 @@ for ( const { title, results } of afterStats ) {
 	 */
 	const rows = [];
 
-	for ( const i in results ) {
-		const newResult = results[ i ];
-
-		for ( const [ metric, values ] of Object.entries( newResult ) ) {
-			// Only do comparison if the number of results is the same.
-			const prevValues =
-				prevStat?.results.length === results.length
-					? prevStat?.results[ i ][ metric ]
-					: null;
-
-			const value = median( values );
-			const prevValue = prevValues ? median( prevValues ) : 0;
-			const delta = value - prevValue;
-			const percentage = ( delta / value ) * 100;
-			const showDiff =
-				metric !== 'wpExtObjCache' && ! Number.isNaN( percentage );
-
-			rows.push( {
-				Metric: metric,
-				Before: formatValue( metric, prevValue ),
-				After: formatValue( metric, value ),
-				'Diff abs.': showDiff ? formatValue( metric, delta ) : '',
-				'Diff %': showDiff ? `${ percentage.toFixed( 2 ) } %` : '',
-				STD: showDiff
-					? formatValue( metric, standardDeviation( values ) )
-					: '',
-				MAD: showDiff
-					? formatValue( metric, medianAbsoluteDeviation( values ) )
-					: '',
-			} );
-		}
+	/**
+	 *
+	 * @type {Record<string, number[]>}
+	 */
+	const metrics = {};
+
+	const newResults = accumulateValues( results );
+	// Only do comparison if the number of results is the same.
+	const prevResults =
+		prevStat && prevStat.results.length === results.length
+			? accumulateValues( prevStat.results )
+			: {};
+
+	for ( const [ metric, values ] of Object.entries( newResults ) ) {
+		const prevValues = prevResults[ metric ] ? prevResults[ metric ] : null;
+
+		const value = median( values );
+		const prevValue = prevValues ? median( prevValues ) : 0;
+		const delta = value - prevValue;
+		const percentage = ( delta / value ) * 100;
+		const showDiff =
+			metric !== 'wpExtObjCache' && ! Number.isNaN( percentage );
+
+		rows.push( {
+			Metric: metric,
+			Before: formatValue( metric, prevValue ),
+			After: formatValue( metric, value ),
+			'Diff abs.': showDiff ? formatValue( metric, delta ) : '',
+			'Diff %': showDiff ? `${ percentage.toFixed( 2 ) } %` : '',
+			STD: showDiff
+				? formatValue( metric, standardDeviation( values ) )
+				: '',
+			MAD: showDiff
+				? formatValue( metric, medianAbsoluteDeviation( values ) )
+				: '',
+		} );
 	}
 
 	console.log( title );

From 81cac1a3cc14b6395c7625a6a077e066a7744abf Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Mon, 29 Apr 2024 10:14:39 +0200
Subject: [PATCH 37/51] More cleanup

---
 .github/workflows/performance.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index a56cc200635c7..73ba8e79d4f13 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -229,6 +229,9 @@ jobs:
       - name: Run any database upgrades
         run: npm run env:cli -- core update-db --path=/var/www/${{ env.LOCAL_DIR }}
 
+      - name: Delete expired transients
+        run: npm run env:cli -- transient delete --expired --path=/var/www/${{ env.LOCAL_DIR }}
+
       - name: Run target performance tests (base/previous commit)
         env:
           TEST_RESULTS_PREFIX: before
@@ -248,6 +251,10 @@ jobs:
         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
         run: npm run env:cli -- core update-db --path=/var/www/${{ env.LOCAL_DIR }}
 
+      - name: Delete expired transients
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
+        run: npm run env:cli -- transient delete --expired --path=/var/www/${{ env.LOCAL_DIR }}
+
       - name: Run baseline performance tests
         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
         env:

From 9a23f401921438f77e5f837c1f1b31877757b91e Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Mon, 29 Apr 2024 12:10:33 +0200
Subject: [PATCH 38/51] Server-Timing in admin

---
 .../wp-content/mu-plugins/server-timing.php   | 42 +++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/tests/performance/wp-content/mu-plugins/server-timing.php b/tests/performance/wp-content/mu-plugins/server-timing.php
index 8c57f31f03314..abf166fcc8e73 100644
--- a/tests/performance/wp-content/mu-plugins/server-timing.php
+++ b/tests/performance/wp-content/mu-plugins/server-timing.php
@@ -49,3 +49,45 @@ static function () use ( $server_timing_values, $template_start, $wpdb ) {
 	},
 	PHP_INT_MAX
 );
+
+add_action(
+	'admin_init',
+	static function () {
+		global $timestart, $wpdb;
+
+		ob_start();
+
+		add_action(
+			'shutdown',
+			static function () use ( $wpdb, $timestart ) {
+				$output = ob_get_clean();
+
+				$server_timing_values = array();
+
+				$server_timing_values['total'] = microtime( true ) - $timestart;
+
+				/*
+				 * While values passed via Server-Timing are intended to be durations,
+				 * any numeric value can actually be passed.
+				 * This is a nice little trick as it allows to easily get this information in JS.
+				 */
+				$server_timing_values['memory-usage']  = memory_get_usage();
+				$server_timing_values['db-queries']    = $wpdb->num_queries;
+				$server_timing_values['ext-obj-cache'] = wp_using_ext_object_cache() ? 1 : 0;
+
+				$header_values = array();
+				foreach ( $server_timing_values as $slug => $value ) {
+					if ( is_float( $value ) ) {
+						$value = round( $value * 1000.0, 2 );
+					}
+					$header_values[] = sprintf( 'wp-%1$s;dur=%2$s', $slug, $value );
+				}
+				header( 'Server-Timing: ' . implode( ', ', $header_values ) );
+
+				echo $output;
+			},
+			PHP_INT_MIN
+		);
+	},
+	PHP_INT_MAX
+);

From 6c4807514a784b1831e7820999a33110bf854f5d Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Mon, 29 Apr 2024 12:45:30 +0200
Subject: [PATCH 39/51] Fix log script

---
 tests/performance/compare-results.js | 22 +------
 tests/performance/log-results.js     | 92 ++++++++++++++++++----------
 tests/performance/utils.js           | 40 ++++++++++++
 3 files changed, 99 insertions(+), 55 deletions(-)

diff --git a/tests/performance/compare-results.js b/tests/performance/compare-results.js
index 0b8ca628df6e7..ea200be1c880c 100644
--- a/tests/performance/compare-results.js
+++ b/tests/performance/compare-results.js
@@ -16,6 +16,7 @@ const {
 	linkToSha,
 	standardDeviation,
 	medianAbsoluteDeviation,
+	accumulateValues,
 } = require( './utils' );
 
 process.env.WP_ARTIFACTS_PATH ??= join( process.cwd(), 'artifacts' );
@@ -91,21 +92,6 @@ if ( process.env.GITHUB_SHA ) {
 	);
 }
 
-/**
- *
- * @param {Array<Record<string, number[]>>} results
- * @returns {Record<string, number[]>}
- */
-function accumulateValues( results ) {
-	return results.reduce( ( acc, result ) => {
-		for ( const [ metric, values ] of Object.entries( result ) ) {
-			acc[ metric ] = acc[ metric ] ?? [];
-			acc[ metric ].push( ...values );
-		}
-		return acc;
-	}, {} );
-}
-
 for ( const { title, results } of afterStats ) {
 	const prevStat = beforeStats.find( ( s ) => s.title === title );
 
@@ -114,12 +100,6 @@ for ( const { title, results } of afterStats ) {
 	 */
 	const rows = [];
 
-	/**
-	 *
-	 * @type {Record<string, number[]>}
-	 */
-	const metrics = {};
-
 	const newResults = accumulateValues( results );
 	// Only do comparison if the number of results is the same.
 	const prevResults =
diff --git a/tests/performance/log-results.js b/tests/performance/log-results.js
index 9ac0fefc9fe4a..66fe1e5291d09 100644
--- a/tests/performance/log-results.js
+++ b/tests/performance/log-results.js
@@ -1,53 +1,77 @@
 #!/usr/bin/env node
 
+/*
+ * Get the test results and format them in the way required by the API.
+ *
+ * Contains some backward compatibility logic for the original test suite format,
+ * see #59900 for details.
+ */
+
 /**
  * External dependencies.
  */
-const fs = require( 'fs' );
-const path = require( 'path' );
 const https = require( 'https' );
 const [ token, branch, hash, baseHash, timestamp, host ] =
 	process.argv.slice( 2 );
-const { median } = require( './utils' );
-
-// The list of test suites to log.
-const testSuites = [
-	'admin',
-	'admin-l10n',
-	'home-block-theme',
-	'home-block-theme-l10n',
-	'home-classic-theme',
-	'home-classic-theme-l10n',
-];
-
-// A list of results to parse based on test suites.
-const testResults = testSuites.map( ( key ) => ( {
-	key,
-	file: `${ key }.test.results.json`,
-} ) );
-
-// A list of base results to parse based on test suites.
-const baseResults = testSuites.map( ( key ) => ( {
-	key,
-	file: `base-${ key }.test.results.json`,
-} ) );
+const { median, parseFile, accumulateValues } = require( './utils' );
+
+const testSuiteMap = {
+	'Admin › Locale: en_US': 'admin',
+	'Admin › Locale: de_DE': 'admin-l10n',
+	'Front End › Theme: twentytwentyone, Locale: en_US': 'home-classic-theme',
+	'Front End › Theme: twentytwentyone, Locale: de_DE':
+		'home-classic-theme-l10n',
+	'Front End › Theme: twentytwentythree, Locale: en_US': 'home-block-theme',
+	'Front End › Theme: twentytwentythree, Locale: de_DE':
+		'home-block-theme-l10n',
+};
 
 /**
- * Parse test files into JSON objects.
- *
- * @param {string} fileName The name of the file.
- * @returns An array of parsed objects from each file.
+ * @type {Array<{file: string, title: string, results: Record<string,number[]>[]}>}
+ */
+const afterStats = parseFile( 'performance-results.json' );
+
+/**
+ * @type {Array<{file: string, title: string, results: Record<string,number[]>[]}>}
+ */
+const baseStats = parseFile( 'base-performance-results.json' );
+
+/**
+ * @type {Record<string, number>}
  */
-const parseFile = ( fileName ) =>
-	JSON.parse(
-		fs.readFileSync( path.join( __dirname, '/specs/', fileName ), 'utf8' )
-	);
+const metrics = {};
+/**
+ * @type {Record<string, number>}
+ */
+const baseMetrics = {};
+
+for ( const { title, results } of afterStats ) {
+	const testSuiteName = testSuiteMap[ title ];
+	if ( ! testSuiteName ) {
+		continue;
+	}
+
+	const baseStat = baseStats.find( ( s ) => s.title === title );
+
+	const currResults = accumulateValues( results );
+	const baseResults = accumulateValues( baseStat.results );
+
+	for ( const [ metric, values ] of Object.entries( currResults ) ) {
+		metrics[ `${ testSuiteName }-${ metric }` ] = median( values );
+	}
+
+	for ( const [ metric, values ] of Object.entries( baseResults ) ) {
+		baseMetrics[ `${ testSuiteName }-${ metric }` ] = median( values );
+	}
+}
+
+process.exit( 0 );
 
 /**
  * Gets the array of metrics from a list of results.
  *
  * @param {Object[]} results A list of results to format.
- * @return {Object[]} Metrics.
+ * @return {Object} Metrics.
  */
 const formatResults = ( results ) => {
 	return results.reduce( ( result, { key, file } ) => {
diff --git a/tests/performance/utils.js b/tests/performance/utils.js
index 9bf6eaa1b0c62..4d023be586048 100644
--- a/tests/performance/utils.js
+++ b/tests/performance/utils.js
@@ -1,3 +1,26 @@
+/**
+ * External dependencies.
+ */
+const { readFileSync, existsSync } = require( 'node:fs' );
+const { join } = require( 'node:path' );
+
+process.env.WP_ARTIFACTS_PATH ??= join( process.cwd(), 'artifacts' );
+
+/**
+ * Parse test files into JSON objects.
+ *
+ * @param {string} fileName The name of the file.
+ * @return {Array<{file: string, title: string, results: Record<string,number[]>[]}>} Parsed object.
+ */
+function parseFile( fileName ) {
+	const file = join( process.env.WP_ARTIFACTS_PATH, fileName );
+	if ( ! existsSync( file ) ) {
+		return [];
+	}
+
+	return JSON.parse( readFileSync( file, 'utf8' ) );
+}
+
 /**
  * Computes the median number from an array numbers.
  *
@@ -141,7 +164,23 @@ function medianAbsoluteDeviation( array = [] ) {
 	return median( array.map( ( a ) => Math.abs( a - med ) ) );
 }
 
+/**
+ *
+ * @param {Array<Record<string, number[]>>} results
+ * @returns {Record<string, number[]>}
+ */
+function accumulateValues( results ) {
+	return results.reduce( ( acc, result ) => {
+		for ( const [ metric, values ] of Object.entries( result ) ) {
+			acc[ metric ] = acc[ metric ] ?? [];
+			acc[ metric ].push( ...values );
+		}
+		return acc;
+	}, {} );
+}
+
 module.exports = {
+	parseFile,
 	median,
 	camelCaseDashes,
 	formatAsMarkdownTable,
@@ -149,4 +188,5 @@ module.exports = {
 	linkToSha,
 	standardDeviation,
 	medianAbsoluteDeviation,
+	accumulateValues,
 };

From bc75856c22734839cc3bf16cf68154baa6eae31c Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Mon, 29 Apr 2024 14:06:46 +0200
Subject: [PATCH 40/51] Undo unrelated change

---
 .github/workflows/pull-request-comments.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/pull-request-comments.yml b/.github/workflows/pull-request-comments.yml
index 34651c9da7588..df276270f1396 100644
--- a/.github/workflows/pull-request-comments.yml
+++ b/.github/workflows/pull-request-comments.yml
@@ -92,7 +92,6 @@ jobs:
       - name: Download artifact
         uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
             const artifacts = await github.rest.actions.listWorkflowRunArtifacts( {
                owner: context.repo.owner,

From a83d9cf3f0bd54f4b9d062e755d4bb5999dc59a6 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Mon, 29 Apr 2024 14:07:50 +0200
Subject: [PATCH 41/51] Don't preserve output

---
 tests/performance/playwright.config.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/performance/playwright.config.js b/tests/performance/playwright.config.js
index c8d18d1e00e5b..c4df0e2872953 100644
--- a/tests/performance/playwright.config.js
+++ b/tests/performance/playwright.config.js
@@ -27,6 +27,7 @@ const config = defineConfig( {
 	timeout: parseInt( process.env.TIMEOUT || '', 10 ) || 600_000, // Defaults to 10 minutes.
 	// Don't report slow test "files", as we will be running our tests in serial.
 	reportSlowTests: null,
+	preserveOutput: 'never',
 	webServer: {
 		...baseConfig.webServer,
 		command: 'npm run env:start',

From 7d0a52a5592ed17d40bbbc4fbfb8276b5b7e7e04 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Mon, 29 Apr 2024 14:14:37 +0200
Subject: [PATCH 42/51] Update docs

---
 .github/workflows/performance.yml | 33 +++++++++++++++----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index 73ba8e79d4f13..3c220063a26e8 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -66,31 +66,33 @@ jobs:
   # - Install WordPress.
   # - Install WordPress Importer plugin.
   # - Import mock data.
+  # - Deactivate WordPress Importer plugin.
   # - Update permalink structure.
+  # - Install additional languages.
+  # - Disable external HTTP requests.
+  # - Disable cron.
+  # - List defined constants.
   # - Install MU plugin.
   # - Run performance tests (current commit).
-  # - Print performance tests results.
-  # - Check out target commit (target branch or previous commit).
-  # - Switch Node.js versions if necessary.
-  # - Install npm dependencies.
-  # - Build WordPress.
+  # - Get previous build (target branch or previous commit).
+  # - Download artifact.
+  # - Unzip the build.
+  # - Flush cache.
   # - Run any database upgrades.
+  # - Delete expired transients.
   # - Run performance tests (previous/target commit).
-  # - Print target performance tests results.
-  # - Reset to original commit.
-  # - Switch Node.js versions if necessary.
-  # - Install npm dependencies.
   # - Set the environment to the baseline version.
+  # - Flush cache.
   # - Run any database upgrades.
+  # - Delete expired transients.
   # - Run baseline performance tests.
-  # - Print baseline performance tests results.
-  # - Compare results with base.
+  # - Archive artifacts.
+  # - Compare results.
   # - Add workflow summary.
   # - Set the base sha.
   # - Set commit details.
   # - Publish performance results.
   # - Ensure version-controlled files are not modified or deleted.
-  # - Dispatch workflow run.
   performance:
     name: Run performance tests ${{ matrix.memcached && '(with memcached)' || '' }}
     runs-on: ubuntu-latest
@@ -232,7 +234,7 @@ jobs:
       - name: Delete expired transients
         run: npm run env:cli -- transient delete --expired --path=/var/www/${{ env.LOCAL_DIR }}
 
-      - name: Run target performance tests (base/previous commit)
+      - name: Run target performance tests (previous/target commit)
         env:
           TEST_RESULTS_PREFIX: before
         run: npm run test:performance
@@ -269,10 +271,7 @@ jobs:
           path: artifacts
           if-no-files-found: ignore
 
-      - name: Ensure version-controlled files are not modified or deleted
-        run: git diff --exit-code
-
-      - name: Compare results with base
+      - name: Compare results
         run: node ./tests/performance/compare-results.js ${{ runner.temp }}/summary.md
 
       - name: Add workflow summary

From 97dcc38b569dce7fe031066e20a9f1a2b963eec8 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Wed, 1 May 2024 12:37:51 +0200
Subject: [PATCH 43/51] Flush after upgrade

---
 .github/workflows/performance.yml | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index 3c220063a26e8..0d39f5dacdb91 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -77,13 +77,13 @@ jobs:
   # - Get previous build (target branch or previous commit).
   # - Download artifact.
   # - Unzip the build.
-  # - Flush cache.
   # - Run any database upgrades.
+  # - Flush cache.
   # - Delete expired transients.
   # - Run performance tests (previous/target commit).
   # - Set the environment to the baseline version.
-  # - Flush cache.
   # - Run any database upgrades.
+  # - Flush cache.
   # - Delete expired transients.
   # - Run baseline performance tests.
   # - Archive artifacts.
@@ -225,12 +225,12 @@ jobs:
           unzip ${{ github.workspace }}/before.zip
           unzip -o ${{ github.workspace }}/wordpress.zip
 
-      - name: Flush cache
-        run: npm run env:cli -- cache flush --path=/var/www/${{ env.LOCAL_DIR }}
-
       - name: Run any database upgrades
         run: npm run env:cli -- core update-db --path=/var/www/${{ env.LOCAL_DIR }}
 
+      - name: Flush cache
+        run: npm run env:cli -- cache flush --path=/var/www/${{ env.LOCAL_DIR }}
+
       - name: Delete expired transients
         run: npm run env:cli -- transient delete --expired --path=/var/www/${{ env.LOCAL_DIR }}
 
@@ -245,14 +245,14 @@ jobs:
           npm run env:cli -- core update --version=${{ env.BASE_TAG }} --force --path=/var/www/${{ env.LOCAL_DIR }}
           npm run env:cli -- core version --path=/var/www/${{ env.LOCAL_DIR }}
 
-      - name: Flush cache
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
-        run: npm run env:cli -- cache flush --path=/var/www/${{ env.LOCAL_DIR }}
-
       - name: Run any database upgrades
         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
         run: npm run env:cli -- core update-db --path=/var/www/${{ env.LOCAL_DIR }}
 
+      - name: Flush cache
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
+        run: npm run env:cli -- cache flush --path=/var/www/${{ env.LOCAL_DIR }}
+
       - name: Delete expired transients
         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
         run: npm run env:cli -- transient delete --expired --path=/var/www/${{ env.LOCAL_DIR }}

From ec092b6c643796e09fe677cde8a3021376315d23 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Wed, 1 May 2024 12:42:07 +0200
Subject: [PATCH 44/51] Don't compare if there is no build

---
 .github/workflows/performance.yml    |  8 +++++++-
 tests/performance/compare-results.js | 20 ++++++++++++--------
 2 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index 0d39f5dacdb91..9772285e1f0eb 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -208,12 +208,13 @@ jobs:
 
       - name: Download artifact
         uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        if: ${{ steps.get-previous-build.outputs.artifact_id || false }}
         with:
           script: |
             const download = await github.rest.actions.downloadArtifact( {
                owner: context.repo.owner,
                repo: context.repo.repo,
-               artifact_id: ${{ steps.get-previous-build.outputs.artifact_id || null }},
+               artifact_id: ${{ steps.get-previous-build.outputs.artifact_id }},
                archive_format: 'zip',
             } );
 
@@ -221,20 +222,25 @@ jobs:
             fs.writeFileSync( '${{ github.workspace }}/before.zip', Buffer.from( download.data ) )
 
       - name: Unzip the build
+        if: ${{ steps.get-previous-build.outputs.artifact_id || false }}
         run: |
           unzip ${{ github.workspace }}/before.zip
           unzip -o ${{ github.workspace }}/wordpress.zip
 
       - name: Run any database upgrades
+        if: ${{ steps.get-previous-build.outputs.artifact_id || false }}
         run: npm run env:cli -- core update-db --path=/var/www/${{ env.LOCAL_DIR }}
 
       - name: Flush cache
+        if: ${{ steps.get-previous-build.outputs.artifact_id || false }}
         run: npm run env:cli -- cache flush --path=/var/www/${{ env.LOCAL_DIR }}
 
       - name: Delete expired transients
+        if: ${{ steps.get-previous-build.outputs.artifact_id || false }}
         run: npm run env:cli -- transient delete --expired --path=/var/www/${{ env.LOCAL_DIR }}
 
       - name: Run target performance tests (previous/target commit)
+        if: ${{ steps.get-previous-build.outputs.artifact_id || false }}
         env:
           TEST_RESULTS_PREFIX: before
         run: npm run test:performance
diff --git a/tests/performance/compare-results.js b/tests/performance/compare-results.js
index ea200be1c880c..c9d51e1a11117 100644
--- a/tests/performance/compare-results.js
+++ b/tests/performance/compare-results.js
@@ -52,14 +52,18 @@ const afterStats = parseFile( 'performance-results.json' );
 let summaryMarkdown = `## Performance Test Results\n\n`;
 
 if ( process.env.TARGET_SHA ) {
-	if ( process.env.GITHUB_SHA ) {
-		summaryMarkdown += `This compares the results from this commit (${ linkToSha(
-			process.env.GITHUB_SHA
-		) }) with the ones from ${ linkToSha( process.env.TARGET_SHA ) }.\n\n`;
+	if ( beforeStats.length > 0 ) {
+		if (process.env.GITHUB_SHA) {
+			summaryMarkdown += `This compares the results from this commit (${linkToSha(
+				process.env.GITHUB_SHA
+			)}) with the ones from ${linkToSha(process.env.TARGET_SHA)}.\n\n`;
+		} else {
+			summaryMarkdown += `This compares the results from this commit with the ones from ${linkToSha(
+				process.env.TARGET_SHA
+			)}.\n\n`;
+		}
 	} else {
-		summaryMarkdown += `This compares the results from this commit with the ones from ${ linkToSha(
-			process.env.TARGET_SHA
-		) }.\n\n`;
+		summaryMarkdown += `Note: no build was found for the target commit ${linkToSha(process.env.TARGET_SHA)}. No comparison is possible.\n\n`;
 	}
 }
 
@@ -119,7 +123,7 @@ for ( const { title, results } of afterStats ) {
 
 		rows.push( {
 			Metric: metric,
-			Before: formatValue( metric, prevValue ),
+			Before: prevValues ? formatValue( metric, prevValue ) : 'N/A',
 			After: formatValue( metric, value ),
 			'Diff abs.': showDiff ? formatValue( metric, delta ) : '',
 			'Diff %': showDiff ? `${ percentage.toFixed( 2 ) } %` : '',

From ffb9387d1740e2e77c95aa2d1c0e6bf2f8d1d416 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Wed, 1 May 2024 12:45:17 +0200
Subject: [PATCH 45/51] Install only Chromium

---
 .github/workflows/performance.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index 9772285e1f0eb..4255d0081af76 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -134,7 +134,7 @@ jobs:
         run: npm ci
 
       - name: Install Playwright browsers
-        run: npx playwright install --with-deps
+        run: npx playwright install --with-deps chromium
 
       - name: Build WordPress
         run: npm run build

From 8feadc8a17d3eb61b8c379537731b7f9de7e78cf Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Wed, 1 May 2024 17:25:58 +0200
Subject: [PATCH 46/51] Add some comments

---
 .github/workflows/performance.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index 4255d0081af76..bc0dcaea1234e 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -182,9 +182,11 @@ jobs:
           npm run env:cli -- language plugin install de_DE --all --path=/var/www/${{ env.LOCAL_DIR }}
           npm run env:cli -- language theme install de_DE --all --path=/var/www/${{ env.LOCAL_DIR }}
 
+      # Prevent background update checks from impacting test stability.
       - name: Disable external HTTP requests
         run: npm run env:cli -- config set WP_HTTP_BLOCK_EXTERNAL true --raw --type=constant --path=/var/www/${{ env.LOCAL_DIR }}
 
+      # Prevent background tasks from impacting test stability.
       - name: Disable cron
         run: npm run env:cli -- config set DISABLE_WP_CRON true --raw --type=constant --path=/var/www/${{ env.LOCAL_DIR }}
 

From 3d3cb50afc06263a4abce7fcb0c24c80b7382890 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Wed, 1 May 2024 17:26:27 +0200
Subject: [PATCH 47/51] Use JS to get the artifact

---
 .github/workflows/performance.yml | 45 ++++++++++++++++++-------------
 1 file changed, 27 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index bc0dcaea1234e..be79476071285 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -201,48 +201,57 @@ jobs:
       - name: Run performance tests (current commit)
         run: npm run test:performance
 
-      - name: Get previous build (target branch or previous commit)
-        id: get-previous-build
-        run: |
-          echo "artifact_id=$(gh api repos/WordPress/wordpress-develop/actions/artifacts?name="wordpress-build-$TARGET_SHA" --jq ".artifacts[] .id")" >> $GITHUB_OUTPUT
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Download artifact
+      - name: Download previous build artifact (target branch or previous commit)
         uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
-        if: ${{ steps.get-previous-build.outputs.artifact_id || false }}
+        id: get-previous-build
+        if: ${{ steps.get-previous-build.outputs.result || false }}
         with:
           script: |
+            const artifacts = await github.rest.actions.listArtifactsForRepo({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              name: "wordpress-build-$TARGET_SHA",
+            });
+
+            const matchArtifact = artifacts.data.artifacts[0];
+
+            if ( ! matchArtifact ) {
+              core.setFailed( 'No artifact found!' );
+              return false;
+            }
+
             const download = await github.rest.actions.downloadArtifact( {
-               owner: context.repo.owner,
-               repo: context.repo.repo,
-               artifact_id: ${{ steps.get-previous-build.outputs.artifact_id }},
-               archive_format: 'zip',
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              artifact_id: matchArtifact.id,
+              archive_format: 'zip',
             } );
 
             const fs = require( 'fs' );
             fs.writeFileSync( '${{ github.workspace }}/before.zip', Buffer.from( download.data ) )
 
+            return true;
+
       - name: Unzip the build
-        if: ${{ steps.get-previous-build.outputs.artifact_id || false }}
+        if: ${{ steps.get-previous-build.outputs.result }}
         run: |
           unzip ${{ github.workspace }}/before.zip
           unzip -o ${{ github.workspace }}/wordpress.zip
 
       - name: Run any database upgrades
-        if: ${{ steps.get-previous-build.outputs.artifact_id || false }}
+        if: ${{ steps.get-previous-build.outputs.result }}
         run: npm run env:cli -- core update-db --path=/var/www/${{ env.LOCAL_DIR }}
 
       - name: Flush cache
-        if: ${{ steps.get-previous-build.outputs.artifact_id || false }}
+        if: ${{ steps.get-previous-build.outputs.result }}
         run: npm run env:cli -- cache flush --path=/var/www/${{ env.LOCAL_DIR }}
 
       - name: Delete expired transients
-        if: ${{ steps.get-previous-build.outputs.artifact_id || false }}
+        if: ${{ steps.get-previous-build.outputs.result }}
         run: npm run env:cli -- transient delete --expired --path=/var/www/${{ env.LOCAL_DIR }}
 
       - name: Run target performance tests (previous/target commit)
-        if: ${{ steps.get-previous-build.outputs.artifact_id || false }}
+        if: ${{ steps.get-previous-build.outputs.result }}
         env:
           TEST_RESULTS_PREFIX: before
         run: npm run test:performance

From 01ef3d48f07455accd4e439da6a7c50422d877a5 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Wed, 1 May 2024 20:01:59 +0200
Subject: [PATCH 48/51] Remove `if`

---
 .github/workflows/performance.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index be79476071285..326dff17c22f0 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -74,7 +74,7 @@ jobs:
   # - List defined constants.
   # - Install MU plugin.
   # - Run performance tests (current commit).
-  # - Get previous build (target branch or previous commit).
+  # - Download previous build artifact (target branch or previous commit).
   # - Download artifact.
   # - Unzip the build.
   # - Run any database upgrades.
@@ -204,7 +204,6 @@ jobs:
       - name: Download previous build artifact (target branch or previous commit)
         uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         id: get-previous-build
-        if: ${{ steps.get-previous-build.outputs.result || false }}
         with:
           script: |
             const artifacts = await github.rest.actions.listArtifactsForRepo({

From 814e2d6ef837f044aaa0f65444d42e1b40fddceb Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Wed, 1 May 2024 21:33:49 +0200
Subject: [PATCH 49/51] Use `process.env`

---
 .github/workflows/performance.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index 326dff17c22f0..e1d85434b1a97 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -209,7 +209,7 @@ jobs:
             const artifacts = await github.rest.actions.listArtifactsForRepo({
               owner: context.repo.owner,
               repo: context.repo.repo,
-              name: "wordpress-build-$TARGET_SHA",
+              name: 'wordpress-build-' . process.env.TARGET_SHA,
             });
 
             const matchArtifact = artifacts.data.artifacts[0];

From 5c91fe8a5756401953d7d447781d1ede6d507141 Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Thu, 2 May 2024 12:16:45 +0200
Subject: [PATCH 50/51] Use correct operator

---
 .github/workflows/performance.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index e1d85434b1a97..6de9cb2c20fa6 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -209,7 +209,7 @@ jobs:
             const artifacts = await github.rest.actions.listArtifactsForRepo({
               owner: context.repo.owner,
               repo: context.repo.repo,
-              name: 'wordpress-build-' . process.env.TARGET_SHA,
+              name: 'wordpress-build-' + process.env.TARGET_SHA,
             });
 
             const matchArtifact = artifacts.data.artifacts[0];

From 9a09bc67e243d49318499f4e7611882fcee6d5ba Mon Sep 17 00:00:00 2001
From: Pascal Birchler <pascalb@google.com>
Date: Thu, 2 May 2024 15:44:31 +0200
Subject: [PATCH 51/51] For now, don't log results of memcached tests

---
 .github/workflows/performance.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index 6de9cb2c20fa6..d79ae5f8012e7 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -295,7 +295,7 @@ jobs:
 
       - name: Set the base sha
         # Only needed when publishing results.
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' && ! matrix.memcached }}
         uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         id: base-sha
         with:
@@ -306,7 +306,7 @@ jobs:
 
       - name: Set commit details
         # Only needed when publishing results.
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' && ! matrix.memcached }}
         uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         id: commit-timestamp
         with:
@@ -317,7 +317,7 @@ jobs:
 
       - name: Publish performance results
         # Only publish results on pushes to trunk.
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' && ! matrix.memcached }}
         env:
             BASE_SHA: ${{ steps.base-sha.outputs.result }}
             COMMITTED_AT: ${{ steps.commit-timestamp.outputs.result }}