Making playwright test work in pipelines (#17941)

* Adding playwright extension to recommended extension

* Fixing args

* Adding gulp commands for vscode

* Adding smoke test steps in pipeline

* Update npm dependencies and gulp commands

* chore: Add init script for Xvfb server

* chore: Add init script for MSSQL server

* chore: Update npm dependencies and gulp commands

* making test work on mac

* chore: Refactor testHelpers.ts to improve code readability and maintainability

* Adding mssql path

* chore: Update mssqlExtensionPath in launchVscodeWithMsSqlExt.ts

* reverting local changes

* syncing to main

* Adding xvfb to gulp

* removing unnecessary files

* chore: update npm test script name

* new way of doing things

* Removing disconnecting button

* chore: Refactor query execution test

* Fixing stuff

* Reverting some changes

* Removing carriage returns

* Removing carriage newlines

* Removing more carriage returns

.gitignore

@@ -9,6 +9,7 @@ packages
 *.nupkg
 *.orig
 *.vsix
+*.trx
 *BROWSE.VC*
 tools
 examples

.vscode/extensions.json

@@ -5,5 +5,6 @@
 		"dbaeumer.vscode-eslint",
 		"EditorConfig.EditorConfig",
 		"GitHub.vscode-pull-request-github",
+		"ms-playwright.playwright"
 	]
 }

build/mssql.init

@@ -0,0 +1,64 @@
+#! /bin/sh -e
+#
+### BEGIN INIT INFO
+# Provides:          sqlserver
+# Required-Start:    $all
+# Required-Stop:
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Manages SQL Server instance on Linux
+### END INIT INFO
+
+DAEMON="/opt/mssql/bin/sqlservr"
+daemon_OPT=""
+DAEMONUSER="mssql"
+daemon_NAME="sqlservr"
+
+export PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:/var/opt/mssql
+
+# Check sqlserver is present
+if [ ! -x $DAEMON ]; then
+        log_failure_msg "$DAEMON not present or not executable"
+        exit 1
+fi
+
+# Load init functions
+. /lib/lsb/init-functions
+
+
+d_start () {
+        log_daemon_msg "Starting system $daemon_NAME Daemon"
+        start-stop-daemon --background --name $daemon_NAME --start --quiet --chuid $DAEMONUSER --exec $DAEMON --umask 007 --oknodo
+        log_end_msg $?
+}
+
+d_stop () {
+        log_daemon_msg "Stopping system $daemon_NAME Daemon"
+        start-stop-daemon --name $daemon_NAME --stop --retry 5 --quiet --name $daemon_NAME --oknodo
+        log_end_msg $?
+}
+
+case "$1" in
+
+        start|stop)
+                d_${1}
+                ;;
+
+        restart|reload|force-reload)
+                        d_stop
+                        d_start
+                ;;
+
+        force-stop)
+               d_stop
+                ;;
+
+        status)
+                status_of_proc "$daemon_NAME" "$DAEMON" "system-wide $daemon_NAME" && exit 0 || exit $?
+                ;;
+        *)
+                echo "Usage: /etc/init.d/$daemon_NAME {start|stop|force-stop|restart|reload|force-reload|status}"
+                exit 1
+                ;;
+esac
+exit 0

build/xvfb.init

@@ -0,0 +1,53 @@
+#!/bin/bash
+#
+# /etc/rc.d/init.d/xvfbd
+#
+# chkconfig: 345 95 28
+# description: Starts/Stops X Virtual Framebuffer server
+# processname: Xvfb
+#
+### BEGIN INIT INFO
+# Provides:          xvfb
+# Required-Start:    $remote_fs $syslog
+# Required-Stop:     $remote_fs $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Start xvfb at boot time
+# Description:       Enable xvfb provided by daemon.
+### END INIT INFO
+
+[ "${NETWORKING}" = "no" ] && exit 0
+
+PROG="/usr/bin/Xvfb"
+PROG_OPTIONS=":10 -ac -screen 0 1024x768x24"
+PROG_OUTPUT="/tmp/Xvfb.out"
+
+case "$1" in
+	start)
+		echo "Starting : X Virtual Frame Buffer "
+		$PROG $PROG_OPTIONS>>$PROG_OUTPUT 2>&1 &
+		disown -ar
+	;;
+	stop)
+		echo "Shutting down : X Virtual Frame Buffer"
+		killproc $PROG
+		RETVAL=$?
+		[ $RETVAL -eq 0 ] && /bin/rm -f /var/lock/subsys/Xvfb
+		/var/run/Xvfb.pid
+		echo
+	;;
+	restart|reload)
+		$0 stop
+		$0 start
+		RETVAL=$?
+	;;
+	status)
+		status Xvfb
+		RETVAL=$?
+	;;
+	*)
+		echo $"Usage: $0 (start|stop|restart|reload|status)"
+		exit 1
+esac
+
+exit $RETVAL

gulpfile.js

@@ -19,6 +19,7 @@ const clc = require('cli-color');
 const path = require('path');
 const esbuild = require('esbuild');
 const { typecheckPlugin } = require('@jgoz/esbuild-plugin-typecheck');
+const run = require('gulp-run-command').default;
 
 require('./tasks/packagetasks');
 require('./tasks/localizationtasks');
@@ -419,6 +420,8 @@ gulp.task('ext:test', async () => {
 	});
 });
 
+gulp.task('ext:smoke', run('npx playwright test'));
+
 gulp.task('test', gulp.series('ext:test'));
 
 require('./tasks/covertasks');

package.json

@@ -53,7 +53,8 @@
     "watch": "gulp watch",
     "lint": "gulp lint",
     "localization": "gulp ext:localization",
-    "test": "gulp test && gulp cover"
+    "smoketest": "gulp ext:smoke",
+    "test": "gulp test:cover --log"
   },
   "devDependencies": {
     "@angular/common": "~2.1.2",
@@ -110,13 +111,15 @@
     "gulp-eslint-new": "^2.1.0",
     "gulp-istanbul-report": "0.0.1",
     "gulp-rename": "^1.2.2",
+    "gulp-run-command": "^0.0.10",
     "gulp-shell": "^0.7.0",
     "gulp-sourcemaps": "^1.6.0",
     "gulp-typescript": "^5.0.1",
     "gulp-uglify": "^2.0.0",
     "husky": "^9.0.11",
     "istanbul": "^0.4.5",
     "npm-run-all": "^4.1.5",
+    "playwright-trx-reporter": "^1.0.8",
     "pm-mocha-jenkins-reporter": "^0.2.6",
     "remap-istanbul": "0.9.6",
     "rxjs": "5.0.0-beta.12",
@@ -169,6 +172,7 @@
     "tar": "^6.2.1",
     "tmp": "^0.0.28",
     "underscore": "^1.8.3",
+    "vsce": "^2.15.0",
     "vscode-languageclient": "5.2.1",
     "vscode-nls": "^2.0.2",
     "zone.js": "^0.6.26"
@@ -241,7 +245,8 @@
           "when": "editorLangId == sql && resourcePath in mssql.runningQueries",
           "group": "navigation@2"
         },
-        {"command": "mssql.connect",
+        {
+          "command": "mssql.connect",
           "when": "editorLangId == sql && resource not in mssql.connections",
           "group": "navigation@3"
         },

playwright.config.ts

@@ -6,7 +6,7 @@
 import dotenv from 'dotenv';
 import { defineConfig } from '@playwright/test';
 
-dotenv.config({ path: './test/e2e/.env' })
+dotenv.config({ path: './test/e2e/.env' });
 
 /**
  * Read environment variables from file.
@@ -29,7 +29,9 @@ export default defineConfig({
   /* Opt out of parallel tests on CI. */
   workers: 1,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  reporter: 'html',
+  reporter: [['playwright-trx-reporter', {
+    outputFile: './test-reports/smoke-results.trx',
+  }]],
   /* Set timeout to 5 minutes */
   timeout: 5 * 60 * 1000,
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
@@ -39,5 +41,6 @@ export default defineConfig({
 
     /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
     trace: 'on-first-retry',
+    video: 'retain-on-failure',
   },
 });

test/e2e/activityBar.spec.ts

@@ -6,6 +6,7 @@
 import { ElectronApplication, Page, test, expect } from '@playwright/test';
 import { launchVsCodeWithMssqlExtension } from './utils/launchVscodeWithMsSqlExt';
 import { screenshotOnFailure } from './utils/screenshotOnError';
+import { mssqlActivityBarButton } from './utils/commonSelectors';
 
 test.describe('MSSQL Extension - Activity Bar', async () => {
 	let vsCodeApp: ElectronApplication;
@@ -18,8 +19,8 @@ test.describe('MSSQL Extension - Activity Bar', async () => {
 	});
 
 	test('MSSQL button is present in activity bar', async () => {
-		await vsCodePage.click('a[aria-label="SQL Server (Ctrl+Alt+D)"]');
-		const count = await vsCodePage.locator('a[aria-label="SQL Server (Ctrl+Alt+D)"]').count();
+		await vsCodePage.click(mssqlActivityBarButton);
+		const count = await vsCodePage.locator(mssqlActivityBarButton).count();
 		expect(count).toEqual(1);
 	});
 

test/e2e/utils/commonSelectors.ts

@@ -0,0 +1,7 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+export const mssqlActivityBarButton = 'a[class^="action-label activity-workbench-view-extension-objectExplorer"]';
+export const addConnectionButton = 'div[aria-label="Add Connection"]';

test/e2e/utils/envConfigReader.ts

@@ -16,7 +16,7 @@ export function getDatabaseName(): string {
 }
 
 export function getAuthenticationType(): string {
-	return process.env.AUTHENTICATION_TYPE ?? 'Integrated'
+	return process.env.AUTHENTICATION_TYPE ?? 'Integrated';
 }
 
 export function getUserName(): string {

test/e2e/utils/launchVscodeWithMsSqlExt.ts

@@ -13,7 +13,7 @@ export async function launchVsCodeWithMssqlExtension(): Promise<{ electronApp: E
 	const vsCodeVersionName = getVsCodeVersionName();
 	const vsCodeExecutablePath = await downloadAndUnzipVSCode(vsCodeVersionName);
 
-		const mssqlExtensionPath = path.resolve(__dirname, '../../../../');
+		const mssqlExtensionPath = path.resolve(__dirname, '../../../');
 		const electronApp = await electron.launch({
 			executablePath: vsCodeExecutablePath,
 			args: [