From 406b5ac0eceaf92a9a29859806cef52847383007 Mon Sep 17 00:00:00 2001 From: ironAiken2 <51399982+ironAiken2@users.noreply.github.com> Date: Fri, 20 Sep 2024 05:34:56 +0000 Subject: [PATCH] test: e2e test for parsing config.toml and modifying (#2705) ### TL;DR Updated e2e tests for config.toml and added TOML parsing functionality. ### What changed? - Removed `config-test.toml` file - Updated `config.test.ts` to include new tests for blocklist and showNonInstalledImages - Modified `test-util.ts` to include a new `modifyConfigToml` function - Added `@iarna/toml` dependency to `package.json` and `pnpm-lock.yaml` The toml parser currently used by webui has a lot of [issues](https://github.com/lablup/backend.ai-webui/issues/2382), so I used a different library, [`@iarna/toml`](https://github.com/iarna/iarna-toml). ### How to test? 1. Run the e2e tests for the config.toml file: - Test the blocklist functionality by checking if specified menu items are hidden and their pages are inaccessible - Test the showNonInstalledImages setting by verifying the visibility of non-installed images 2. Verify that the `modifyConfigToml` function works correctly by running the tests and checking if the config.toml file is modified as expected ### Why make this change? This change improves the e2e testing capabilities for the config.toml file, allowing for more dynamic and comprehensive testing of configuration options. The addition of TOML parsing functionality enables easier manipulation of the config file during tests, providing a more flexible and maintainable testing approach. --- **Checklist:** (if applicable) - [ ] Mention to the original issue - [ ] Documentation - [ ] Minium required manager version - [ ] Specific setting for review (eg., KB link, endpoint or how to setup) - [ ] Minimum requirements to check during review - [ ] Test case(s) to demonstrate the difference of before/after --- e2e/config-test.toml | 79 -------------- e2e/config.test.ts | 255 +++++++++++++++++++++++++++++-------------- e2e/test-util.ts | 38 ++++++- package.json | 3 +- pnpm-lock.yaml | 8 ++ 5 files changed, 213 insertions(+), 170 deletions(-) delete mode 100644 e2e/config-test.toml diff --git a/e2e/config-test.toml b/e2e/config-test.toml deleted file mode 100644 index b638af57ca..0000000000 --- a/e2e/config-test.toml +++ /dev/null @@ -1,79 +0,0 @@ -[general] -# apiEndpoint = "http://127.0.0.1:8090" -apiEndpointText = "Backend.AI Admin Training" - -defaultSessionEnvironment = "cr.backend.ai/stable/python-tensorflow" - -maxCountForPreOpenedPort = 1 -# siteDescription = "" -connectionMode = "SESSION" #connectionMode = API -allowChangeSigninMode = true -signupSupport = true -allowSignout = true -allowAnonymousChangePassword = true -allowProjectResourceMonitor = true -allowManualImageNameForSession = true -allowSignupWithoutConfirmation = true -allowCustomResourceAllocation = false -autoLogout = false -# loginAttemptLimit = 1 -# loginBlockTime = 30 -debug = false -maskUserInfo = true -enableContainerCommit = true -enablePipeline = true -hideAgents = true -alwaysEnqueueComputeSession = true -# If false, show the `Agent Summary` menu in the sidebar. -showCustomResourceAllocation = true -enable2FA = true -force2FA = false -systemSSHImage = "cr.backend.ai/stable/python-tensorflow" -isDirectorySizeVisible = false -allowCustomResourceAllocation = true -forceTerminateTimeThreshold = 259200 -enableModelStore = true -enableExtendLoginSession = false - -[wsproxy] -proxyURL = "http://127.0.0.1:5050/" -autoPortSelection = true -disableCertCheck = true - -[resources] -openPortToPublic = true -# Show option to open app proxy port to anyone. -maxCPUCoresPerContainer = 256 # Maximum CPU per container. -maxMemoryPerContainer = 64 # Maximum memory per container. -maxCUDADevicesPerContainer = 16 # Maximum CUDA devices per container. -maxCUDASharesPerContainer = 16 # Maximum CUDA shares per container. -maxROCMDevicesPerContainer = 10 # Maximum ROCm devices per container. -maxTPUDevicesPerContainer = 8 # Maximum TPU devices per container. -maxIPUDevicesPerContainer = 8 # Maximum IPU devices per container. -maxATOMDevicesPerContainer = 8 # Maximum ATOM devices per container. -maxWarboyDevicesPerContainer = 8 # Maximum Warboy devices per container. -maxShmPerContainer = 16 # Maximum shared memory per container. -maxFileUploadSize = 4294967296 # Maximum size of single file upload. Set to -1 for unlimited upload. -allowPreferredPort = true -allowNonAuthTCP = true - -[menu] -# blocklist = "pipeline,summary" -#blocklist = "pipeline,serving" # "statistics" -#blocklist = "" # Hide menus on the list which are separated by comma. please note that summary menu cannot be added to blocklist -#inactivelist = "statistics,pipeline,serving" # Disable menus on the list which are separated by comma. - -[server] -# webServerURL = - -[environments] -#allowlist = "" -showNonInstalledImages = true - -[plugin] -# Reserved to load plugins - -[pipeline] -#endpoint = "http://mlops.com:9500" # FastTrack endpoint. -frontendEndpoint = "http://127.0.0.1:9500" -hideSideMenuButton = false diff --git a/e2e/config.test.ts b/e2e/config.test.ts index 6399aef7b5..b3791ecfe6 100644 --- a/e2e/config.test.ts +++ b/e2e/config.test.ts @@ -1,87 +1,174 @@ -import { loginAsAdmin, mockConfigToml, webuiEndpoint } from './test-util'; +import { loginAsAdmin, modifyConfigToml, webuiEndpoint } from './test-util'; import { test, expect } from '@playwright/test'; -test.describe('config read and test', () => { - test('showNonInstalledImages', async ({ page, context }) => { - // manipulate config file from mocked one - await mockConfigToml(page, './config-test.toml'); - - await loginAsAdmin(page); - - await page.getByRole('group').getByText('Environments').click(); - await page - .getByRole('columnheader', { name: 'Status' }) - .locator('div') - .click(); - await page - .getByRole('columnheader', { name: 'Status' }) - .locator('div') - .click(); - - const registryCell = await page - .locator('.ant-table-row > td:nth-child(3)') - .first(); - const registry = await registryCell.textContent(); - - const architectureCell = await page - .locator('.ant-table-row > td:nth-child(4)') - .first(); - const architecture = await architectureCell.textContent(); - - const namespaceCell = await page - .getByRole('cell', { name: 'community' }) - .first(); - const namespace = await namespaceCell.textContent(); - - const languageCell = await page.getByRole('cell', { name: 'afni' }).first(); - const language = await languageCell.textContent(); - - const versionCell = await page - .getByRole('cell', { name: 'ubuntu18.04' }) - .first(); - const version = await versionCell.textContent(); - - const uninstalledImageString = `${registry}/${namespace}/${language}:${version}@${architecture}`; - - await page.goto(webuiEndpoint); - await page.getByLabel('power_settings_new').click(); - await page - .getByRole('button', { name: '2 Environments & Resource' }) - .click(); - await page - .locator( - '.ant-form-item-control-input-content > .ant-select > .ant-select-selector', - ) - .first() - .click(); - await page.getByLabel('Environments / Version').fill('AF'); - await page - .locator('.rc-virtual-list-holder-inner > div:nth-child(2)') - .click(); - - await page - .locator('span') - .filter({ hasText: 'ubuntu18.04x86_64' }) - .locator('div') - .first() - .click(); - await page - .locator( - 'div:nth-child(4) > .rc-virtual-list-holder > div > .rc-virtual-list-holder-inner > .ant-select-item > .ant-select-item-option-content > div', - ) - .click(); - await page - .getByRole('button', { name: 'Skip to review double-right' }) - .click(); - await page.getByRole('button', { name: 'Copy' }).click(); - - await context.grantPermissions(['clipboard-read', 'clipboard-write']); - - const handle = await page.evaluateHandle(() => - navigator.clipboard.readText(), - ); - const clipboardContent = await handle.jsonValue(); - - expect(clipboardContent).toEqual(uninstalledImageString); - }); +test.describe.parallel('config.toml', () => { + test( + 'enableLLMPlayground', + { tag: ['@serving'] }, + async ({ page, request }) => { + // modify config.toml to enable LLM playground + let requestConfig = { + general: { + enableLLMPlayground: true, + }, + }; + await modifyConfigToml(page, request, requestConfig); + await loginAsAdmin(page); + await page.getByRole('menuitem', { name: 'Serving' }).click(); + await expect( + page.getByRole('tab', { name: 'LLM Playground' }), + ).toBeVisible(); + + requestConfig.general.enableLLMPlayground = false; + await modifyConfigToml(page, request, requestConfig); + await page.reload(); + await page.getByRole('menuitem', { name: 'Serving' }).click(); + await expect( + page.getByRole('tab', { name: 'LLM Playground' }), + ).toBeHidden(); + }, + ); + + test( + 'block list', + { tag: ['@session', '@summary', '@serving'] }, + async ({ page, request }) => { + // modify config.toml to blocklist some menu items + let requestConfig = { + menu: { + blocklist: 'summary, serving, job', + }, + }; + await modifyConfigToml(page, request, requestConfig); + await loginAsAdmin(page); + + // check if the menu items are hidden + await expect( + page.getByRole('menuitem', { name: 'Summary' }), + ).toBeHidden(); + await expect( + page.getByRole('menuitem', { name: 'Sessions' }), + ).toBeHidden(); + await expect( + page.getByRole('menuitem', { name: 'Serving' }), + ).toBeHidden(); + + // check if the pages are not accessible + await page.goto(`${webuiEndpoint}/summary`); + await expect(page).toHaveURL(/.*error/); + await page.goto(`${webuiEndpoint}/serving`); + await expect(page).toHaveURL(/.*error/); + await page.goto(`${webuiEndpoint}/job`); + await expect(page).toHaveURL(/.*error/); + + requestConfig.menu.blocklist = ''; + await modifyConfigToml(page, request, requestConfig); + await page.reload(); + + // check if the menu items are visible + await expect( + page.getByRole('menuitem', { name: 'Summary' }), + ).toBeVisible(); + await expect( + page.getByRole('menuitem', { name: 'Sessions' }), + ).toBeVisible(); + await expect( + page.getByRole('menuitem', { name: 'Serving' }), + ).toBeVisible(); + }, + ); + + test( + 'showNonInstalledImages', + { tag: ['@session'] }, + async ({ page, context, request }) => { + // modify config.toml to show non-installed images + const requestConfig = { + environments: { + showNonInstalledImages: true, + }, + }; + await modifyConfigToml(page, request, requestConfig); + + await loginAsAdmin(page); + + await page.getByRole('group').getByText('Environments').click(); + await page + .getByRole('columnheader', { name: 'Status' }) + .locator('div') + .click(); + await page + .getByRole('columnheader', { name: 'Status' }) + .locator('div') + .click(); + + const registryCell = await page + .locator('.ant-table-row > td:nth-child(3)') + .first(); + const registry = await registryCell.textContent(); + + const architectureCell = await page + .locator('.ant-table-row > td:nth-child(4)') + .first(); + const architecture = await architectureCell.textContent(); + + const namespaceCell = await page + .getByRole('cell', { name: 'community' }) + .first(); + const namespace = await namespaceCell.textContent(); + + const languageCell = await page + .getByRole('cell', { name: 'afni' }) + .first(); + const language = await languageCell.textContent(); + + const versionCell = await page + .getByRole('cell', { name: 'ubuntu18.04' }) + .first(); + const version = await versionCell.textContent(); + + const uninstalledImageString = `${registry}/${namespace}/${language}:${version}@${architecture}`; + + await page.goto(webuiEndpoint); + await page.getByLabel('power_settings_new').click(); + await page + .getByRole('button', { name: '2 Environments & Resource' }) + .click(); + await page + .locator( + '.ant-form-item-control-input-content > .ant-select > .ant-select-selector', + ) + .first() + .click(); + await page.getByLabel('Environments / Version').fill('AF'); + await page + .locator('.rc-virtual-list-holder-inner > div:nth-child(2)') + .click(); + + await page + .locator('span') + .filter({ hasText: 'ubuntu18.04x86_64' }) + .locator('div') + .first() + .click(); + await page + .locator( + 'div:nth-child(4) > .rc-virtual-list-holder > div > .rc-virtual-list-holder-inner > .ant-select-item > .ant-select-item-option-content > div', + ) + .click(); + await page + .getByRole('button', { name: 'Skip to review double-right' }) + .click(); + await page.getByRole('button', { name: 'Copy' }).click(); + + await context.grantPermissions(['clipboard-read', 'clipboard-write']); + + const handle = await page.evaluateHandle(() => + navigator.clipboard.readText(), + ); + const clipboardContent = await handle.jsonValue(); + + expect(clipboardContent).toEqual(uninstalledImageString); + }, + ); }); diff --git a/e2e/test-util.ts b/e2e/test-util.ts index 81bd9f1c80..7343ffa20d 100644 --- a/e2e/test-util.ts +++ b/e2e/test-util.ts @@ -1,4 +1,12 @@ -import { Locator, Page, expect } from '@playwright/test'; +import TOML from '@iarna/toml'; +import { + APIRequestContext, + Locator, + Page, + Request, + expect, + request, +} from '@playwright/test'; import fs from 'fs/promises'; import path from 'path'; @@ -229,14 +237,32 @@ export async function deleteSession(page: Page, sessionName: string) { await expect(page.getByText(sessionName)).toBeHidden(); } -export async function mockConfigToml(page: Page, rawPath) { - const filePath = path.resolve(__dirname, rawPath); - const mockData = await fs.readFile(filePath, 'utf-8'); - await page.route('http://127.0.0.1:9081/config.toml', async (route) => { +/** + * Modify specific columns in the webui config.toml file + * + * @param page + * @param request + * @param configColumn + * The object to modify the config.toml file + * + * e.g. { "environments": { "showNonInstalledImages": "true" } } + */ +export async function modifyConfigToml( + page: Page, + request: APIRequestContext, + configColumn: Record>, +) { + const configToml = await ( + await request.get(`${webuiEndpoint}/config.toml`) + ).text(); + const config = TOML.parse(configToml); + Object.assign(config, configColumn); + + await page.route(`${webuiEndpoint}/config.toml`, async (route) => { await route.fulfill({ status: 200, contentType: 'text/plain', - body: mockData, + body: TOML.stringify(config), }); }); } diff --git a/package.json b/package.json index 5ddd4b265b..4793134268 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "theme-schema:update": "pnpm dlx typescript-json-schema \"./react/node_modules/antd/es/config-provider/context.d.ts\" ThemeConfig -o ./resources/antdThemeConfig.schema.json --esModuleInterop" }, "dependencies": { + "@iarna/toml": "^2.2.5", "@lit/reactive-element": "^2.0.4", "@material/mwc-button": "^0.27.0", "@material/mwc-checkbox": "^0.27.0", @@ -188,7 +189,7 @@ ], "src/**/*.{js,jsx,ts,tsx,json,css,scss,md}": [ "prettier --write" - ], + ], "e2e/**/*.{js,ts}": [ "prettier --write" ] diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 12b6d04aab..48f0dc13bc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: dependencies: + '@iarna/toml': + specifier: ^2.2.5 + version: 2.2.5 '@lit/reactive-element': specifier: ^2.0.4 version: 2.0.4 @@ -1328,6 +1331,9 @@ packages: resolution: {integrity: sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==} deprecated: Use @eslint/object-schema instead + '@iarna/toml@2.2.5': + resolution: {integrity: sha512-trnsAYxU3xnS1gPHPyU961coFyLkh4gAD/0zQ5mymY4yOZ+CYvsPqUbOFSw0aDM4y0tV7tiFxL/1XfXPNC6IPg==} + '@isaacs/cliui@8.0.2': resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} @@ -8562,6 +8568,8 @@ snapshots: '@humanwhocodes/object-schema@2.0.3': {} + '@iarna/toml@2.2.5': {} + '@isaacs/cliui@8.0.2': dependencies: string-width: 5.1.2