From b01a3fc4c624853039c2a173cc74a3912074bca1 Mon Sep 17 00:00:00 2001 From: Adir111 Date: Sun, 10 Nov 2024 17:37:32 +0200 Subject: [PATCH 01/14] fixed bug --- core/api-server/lib/service/pipeline-versions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/api-server/lib/service/pipeline-versions.js b/core/api-server/lib/service/pipeline-versions.js index 4496e8995..c4f8b1f7a 100644 --- a/core/api-server/lib/service/pipeline-versions.js +++ b/core/api-server/lib/service/pipeline-versions.js @@ -14,7 +14,7 @@ class PipelineVersions { } async getVersions(options) { - validator.pipelines.validatePipelineName(options); + validator.pipelines.validatePipelineName(options.name); return versioning.getVersions(options, true); } From cde3ed07f9b2358963e30c1eb2f927d1aa1befcd Mon Sep 17 00:00:00 2001 From: Adir111 Date: Sun, 10 Nov 2024 17:38:03 +0200 Subject: [PATCH 02/14] changed file name --- core/api-server/tests/{versions.js => algorithms-versions.js} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename core/api-server/tests/{versions.js => algorithms-versions.js} (100%) diff --git a/core/api-server/tests/versions.js b/core/api-server/tests/algorithms-versions.js similarity index 100% rename from core/api-server/tests/versions.js rename to core/api-server/tests/algorithms-versions.js From f8ec0ed5c7d840ee753722d103cedbd169702ec9 Mon Sep 17 00:00:00 2001 From: Adir111 Date: Sun, 10 Nov 2024 18:27:56 +0200 Subject: [PATCH 03/14] fixed bug --- core/api-server/lib/service/pipeline-versions.js | 2 +- core/api-server/lib/service/pipelines.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/api-server/lib/service/pipeline-versions.js b/core/api-server/lib/service/pipeline-versions.js index c4f8b1f7a..8756dc7ea 100644 --- a/core/api-server/lib/service/pipeline-versions.js +++ b/core/api-server/lib/service/pipeline-versions.js @@ -23,7 +23,7 @@ class PipelineVersions { if (!pipeline) { throw new ResourceNotFoundError('pipeline', name); } - const pipelineVersion = await versioning.getVersion({ version }); + const pipelineVersion = await versioning.getVersion({ version }, true); if (!pipelineVersion) { throw new ResourceNotFoundError('version', version); } diff --git a/core/api-server/lib/service/pipelines.js b/core/api-server/lib/service/pipelines.js index d73be8333..50a816e37 100644 --- a/core/api-server/lib/service/pipelines.js +++ b/core/api-server/lib/service/pipelines.js @@ -244,7 +244,7 @@ class PipelineService { const version = await this._versioning(true, newPipeline); newPipeline.version = version; await stateManager.insertPipeline(newPipeline); - return options; + return newPipeline; } _comparePipelines(oldPipeline, newPipeline) { From 8ccac0b5d2c31455a98c8c1f7906acdaba3b3d7b Mon Sep 17 00:00:00 2001 From: Adir111 Date: Sun, 10 Nov 2024 19:10:26 +0200 Subject: [PATCH 04/14] bug fix --- core/api-server/lib/service/versioning.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/api-server/lib/service/versioning.js b/core/api-server/lib/service/versioning.js index 75ae5a688..69eaffd1a 100644 --- a/core/api-server/lib/service/versioning.js +++ b/core/api-server/lib/service/versioning.js @@ -17,7 +17,7 @@ class Versioning { const extractedObject = isPipeline ? object.pipeline : object.algorithm; const { name } = extractedObject; const version = uid({ length: SETTINGS.VERSION_LENGTH }); - const latestSemver = await this.getLatestSemver({ name }); + const latestSemver = await this.getLatestSemver({ name }, isPipeline); const semver = this.incSemver(latestSemver); const newVersion = { version, From 54f763302f47fd854aeb20a4589b72a96918c678 Mon Sep 17 00:00:00 2001 From: Adir111 Date: Sun, 10 Nov 2024 19:42:38 +0200 Subject: [PATCH 05/14] added tests for pipelines versioning --- core/api-server/tests/pipelines-versions.js | 107 ++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 core/api-server/tests/pipelines-versions.js diff --git a/core/api-server/tests/pipelines-versions.js b/core/api-server/tests/pipelines-versions.js new file mode 100644 index 000000000..6175feb4c --- /dev/null +++ b/core/api-server/tests/pipelines-versions.js @@ -0,0 +1,107 @@ +const { expect } = require('chai'); +const { StatusCodes } = require('http-status-codes'); +const clone = require('clone'); +const { pipelineStatuses, nodeKind } = require('@hkube/consts'); +const { uid: uuid } = require('@hkube/uid'); +const { pipelines: [pipeline] } = require('./mocks'); +const { request } = require('./utils'); +const stateManager = require('../lib/state/state-manager'); +let restUrl, restPath; + +describe.only('Versions/Pipelines', () => { + const pipeList = []; + + const addPipeline = async (pipeline) => { + const name = `pipe-test-${uuid()}`; + await stateManager.deletePipeline({ name, keepOldVersions: false }) + const addRequest = { uri: `${restUrl}/store/pipelines`, method: 'POST', body: pipeline }; + pipeline.name = name; + const res = await request(addRequest); + pipeList.push(name); + return { name, version: res.body.version }; + } + + const updatePipeline = async (pipeline) => { + const updateRequest = { uri: `${restUrl}/store/pipelines`, method: 'PUT', body: pipeline }; + await request(updateRequest); + } + + const getAllVersions = async (name) => { + const versionReq = { uri: `${restPath}/${name}`, method: 'GET' }; + const res = await request(versionReq); + return res.body; + } + + const getSpecificVersion = async (name, version) => { + const versionReq = { uri: `${restPath}/${name}/${version}`, method: 'GET' }; + const res = await request(versionReq); + return res.body.version; + } + + before(() => { + restUrl = global.testParams.restUrl; + restPath = `${restUrl}/versions/pipelines`; + }); + + beforeEach(function () { + console.log('\n-----------------------------------------------\n'); + }); + + after(async function () { + this.timeout(2 * 60 * 1000); + console.log("pipeList = " + pipeList); + j = 0; + z = 3; + + while (j < pipeList.length) { + delPipe = pipeList.slice(j, z); + const del = delPipe.map((e) => { + return stateManager.deletePipelines(delPipe); + }) + console.log("delPipe-", JSON.stringify(delPipe, null, 2)); + const delResult = await Promise.all(del); + delResult.forEach(result => { + if (result && result.text) { + console.log("Delete Result Message:", result.text); + } + }); + j += 3; + z += 3; + console.log("j=" + j + ",z=" + z); + } + console.log("----------------------- end -----------------------"); + }); + + describe('get', () => { + it('should succeed to get list of the new pipeline version', async () => { + const { name } = await addPipeline(pipeline); + const versionsList = await getAllVersions(name); + expect(versionsList).to.have.lengthOf(1); + }); + + it('should succeed to get version', async () => { + const { name, version } = await addPipeline(pipeline); + const specificVersion = await getSpecificVersion(name, version); + expect(specificVersion).to.eql(version); + }); + + it('should succeed to get versions', async () => { + const { name } = await addPipeline(pipeline); + const pipeline2 = clone(pipeline); + pipeline2.options.ttl = 6666; + await updatePipeline(pipeline2); + const versionsList = await getAllVersions(name) + const semver = versionsList.map((v) => v.semver); + expect(versionsList).to.have.lengthOf(2); + expect(semver).to.eql(['1.0.1', '1.0.0']); + }); + }); + + describe.skip('versions when pipeline is deleted', () => { + it('should return empty list after pipeline deleted', async () => { + const { name } = await addPipeline(pipeline); + await stateManager.deletePipeline({ name, keepOldVersions: false }); + + }) + }); +}); From d0486159b2d2e2d55dbb99fed617f97990efddec Mon Sep 17 00:00:00 2001 From: Adir111 Date: Mon, 11 Nov 2024 08:28:16 +0200 Subject: [PATCH 06/14] added deletion tests --- core/api-server/tests/pipelines-versions.js | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/core/api-server/tests/pipelines-versions.js b/core/api-server/tests/pipelines-versions.js index 6175feb4c..27efe1135 100644 --- a/core/api-server/tests/pipelines-versions.js +++ b/core/api-server/tests/pipelines-versions.js @@ -1,7 +1,5 @@ const { expect } = require('chai'); -const { StatusCodes } = require('http-status-codes'); const clone = require('clone'); -const { pipelineStatuses, nodeKind } = require('@hkube/consts'); const { uid: uuid } = require('@hkube/uid'); const { pipelines: [pipeline] } = require('./mocks'); const { request } = require('./utils'); @@ -97,11 +95,19 @@ describe.only('Versions/Pipelines', () => { }); }); - describe.skip('versions when pipeline is deleted', () => { + describe('versions when pipeline is deleted', () => { it('should return empty list after pipeline deleted', async () => { const { name } = await addPipeline(pipeline); await stateManager.deletePipeline({ name, keepOldVersions: false }); - - }) + const versionsList = await getAllVersions(name); + expect(versionsList).to.have.lengthOf(0); + }); + + it('should return the versions of the deleted pipeline', async () => { + const { name } = await addPipeline(pipeline); + await stateManager.deletePipeline({ name, keepOldVersions: true }); + const versionsList = await getAllVersions(name); + expect(versionsList).to.have.lengthOf(1); + }); }); }); From 9e3390abf5a3dd48a8c56f8c98ba66d645866b0c Mon Sep 17 00:00:00 2001 From: Adir111 Date: Mon, 11 Nov 2024 08:29:27 +0200 Subject: [PATCH 07/14] removed deletion --- core/api-server/tests/pipelines-versions.js | 37 +++------------------ 1 file changed, 4 insertions(+), 33 deletions(-) diff --git a/core/api-server/tests/pipelines-versions.js b/core/api-server/tests/pipelines-versions.js index 27efe1135..2838fd07b 100644 --- a/core/api-server/tests/pipelines-versions.js +++ b/core/api-server/tests/pipelines-versions.js @@ -1,13 +1,14 @@ const { expect } = require('chai'); const clone = require('clone'); const { uid: uuid } = require('@hkube/uid'); -const { pipelines: [pipeline] } = require('./mocks'); +const { pipelines } = require('./mocks'); const { request } = require('./utils'); const stateManager = require('../lib/state/state-manager'); let restUrl, restPath; -describe.only('Versions/Pipelines', () => { - const pipeList = []; + +describe('Versions/Pipelines', () => { + const pipeline = clone(pipelines[0]); const addPipeline = async (pipeline) => { const name = `pipe-test-${uuid()}`; @@ -15,7 +16,6 @@ describe.only('Versions/Pipelines', () => { const addRequest = { uri: `${restUrl}/store/pipelines`, method: 'POST', body: pipeline }; pipeline.name = name; const res = await request(addRequest); - pipeList.push(name); return { name, version: res.body.version }; } @@ -41,35 +41,6 @@ describe.only('Versions/Pipelines', () => { restPath = `${restUrl}/versions/pipelines`; }); - beforeEach(function () { - console.log('\n-----------------------------------------------\n'); - }); - - after(async function () { - this.timeout(2 * 60 * 1000); - console.log("pipeList = " + pipeList); - j = 0; - z = 3; - - while (j < pipeList.length) { - delPipe = pipeList.slice(j, z); - const del = delPipe.map((e) => { - return stateManager.deletePipelines(delPipe); - }) - console.log("delPipe-", JSON.stringify(delPipe, null, 2)); - const delResult = await Promise.all(del); - delResult.forEach(result => { - if (result && result.text) { - console.log("Delete Result Message:", result.text); - } - }); - j += 3; - z += 3; - console.log("j=" + j + ",z=" + z); - } - console.log("----------------------- end -----------------------"); - }); - describe('get', () => { it('should succeed to get list of the new pipeline version', async () => { const { name } = await addPipeline(pipeline); From 6c7105c277ba0d1cf14fa6a7139246842116be92 Mon Sep 17 00:00:00 2001 From: Adir111 Date: Mon, 11 Nov 2024 08:30:25 +0200 Subject: [PATCH 08/14] fixed test --- core/api-server/tests/exec-streaming.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/api-server/tests/exec-streaming.js b/core/api-server/tests/exec-streaming.js index 66ffb8cec..534a401fd 100644 --- a/core/api-server/tests/exec-streaming.js +++ b/core/api-server/tests/exec-streaming.js @@ -850,6 +850,10 @@ describe('Streaming', () => { }; const response = await request(options); expect(response.response.statusCode).to.equal(HttpStatus.StatusCodes.CREATED); + expect(response.body).to.have.property('modified'); + expect(response.body).to.have.property('version'); + delete response.body.version; + delete response.body.modified; expect(response.body).to.deep.equal(pipeline); }); }); From a5d4edfe731f10b7c637d10b37752904cf85c03a Mon Sep 17 00:00:00 2001 From: Adir111 Date: Mon, 11 Nov 2024 08:31:25 +0200 Subject: [PATCH 09/14] fixed test --- core/api-server/tests/pipelines-store.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/api-server/tests/pipelines-store.js b/core/api-server/tests/pipelines-store.js index 85549d2f3..f17718726 100644 --- a/core/api-server/tests/pipelines-store.js +++ b/core/api-server/tests/pipelines-store.js @@ -403,6 +403,10 @@ describe('Store/Pipelines', () => { }; const response = await request(options); expect(response.response.statusCode).to.equal(StatusCodes.CREATED); + expect(response.body).to.have.property('modified'); + expect(response.body).to.have.property('version'); + delete response.body.version; + delete response.body.modified; expect(response.body).to.eql(pipeline); const storedPipeline = await request({ uri: restPath + '/' + pipeline.name, From 56b23ac3f5662edf2956e71299d045e681ae602e Mon Sep 17 00:00:00 2001 From: Adir111 Date: Mon, 11 Nov 2024 08:37:14 +0200 Subject: [PATCH 10/14] moved line --- core/api-server/lib/service/pipelines.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/api-server/lib/service/pipelines.js b/core/api-server/lib/service/pipelines.js index 50a816e37..e6af0bfaf 100644 --- a/core/api-server/lib/service/pipelines.js +++ b/core/api-server/lib/service/pipelines.js @@ -13,9 +13,9 @@ const versionsService = require('./pipeline-versions'); class PipelineService { async updatePipeline(options) { validator.pipelines.validateUpdatePipeline(options); - const oldPipeLine = await this.getPipeline(options); await validator.algorithms.validateAlgorithmExists(options); validator.gateways.validateGatewayNodes(options.nodes); + const oldPipeLine = await this.getPipeline(options); const newPipeline = { modified: Date.now(), ...options, From 5a2f2649eef95da083c1ff4b8b5d57b9118fa4db Mon Sep 17 00:00:00 2001 From: Adir111 Date: Mon, 11 Nov 2024 08:43:46 +0200 Subject: [PATCH 11/14] return new pipeline when updating pipeline --- core/api-server/lib/service/pipelines.js | 2 +- core/api-server/tests/pipelines-store.js | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/core/api-server/lib/service/pipelines.js b/core/api-server/lib/service/pipelines.js index e6af0bfaf..39af36356 100644 --- a/core/api-server/lib/service/pipelines.js +++ b/core/api-server/lib/service/pipelines.js @@ -26,7 +26,7 @@ class PipelineService { newPipeline.version = newVersion; } await stateManager.replacePipeline(newPipeline); - return options; + return newPipeline; } async deletePipeline(options) { diff --git a/core/api-server/tests/pipelines-store.js b/core/api-server/tests/pipelines-store.js index f17718726..f811c45df 100644 --- a/core/api-server/tests/pipelines-store.js +++ b/core/api-server/tests/pipelines-store.js @@ -743,6 +743,10 @@ describe('Store/Pipelines', () => { body: pipeline }; const response = await request(options); + expect(response.body).to.have.property('modified'); + expect(response.body).to.have.property('version'); + delete response.body.version; + delete response.body.modified; expect(response.body).to.deep.equal(pipeline); }); it('should throw validation error if algorithmName not exists', async () => { From 63ab9e238be4301446506d4e6252442df67cbfab Mon Sep 17 00:00:00 2001 From: Adir111 Date: Mon, 11 Nov 2024 08:49:07 +0200 Subject: [PATCH 12/14] updated test --- core/api-server/tests/pipelines-versions.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/core/api-server/tests/pipelines-versions.js b/core/api-server/tests/pipelines-versions.js index 2838fd07b..a25cde928 100644 --- a/core/api-server/tests/pipelines-versions.js +++ b/core/api-server/tests/pipelines-versions.js @@ -21,7 +21,8 @@ describe('Versions/Pipelines', () => { const updatePipeline = async (pipeline) => { const updateRequest = { uri: `${restUrl}/store/pipelines`, method: 'PUT', body: pipeline }; - await request(updateRequest); + const res = await request(updateRequest); + return res.body.version; } const getAllVersions = async (name) => { @@ -54,15 +55,17 @@ describe('Versions/Pipelines', () => { expect(specificVersion).to.eql(version); }); - it('should succeed to get versions', async () => { - const { name } = await addPipeline(pipeline); + it.only('should succeed to get versions and change version to latest', async () => { + const { name, version: oldVersion } = await addPipeline(pipeline); const pipeline2 = clone(pipeline); pipeline2.options.ttl = 6666; - await updatePipeline(pipeline2); + const newVersion = await updatePipeline(pipeline2); const versionsList = await getAllVersions(name) const semver = versionsList.map((v) => v.semver); expect(versionsList).to.have.lengthOf(2); expect(semver).to.eql(['1.0.1', '1.0.0']); + expect(oldVersion).to.be.not.equal(newVersion); + expect(newVersion).to.be.equal(versionsList[0].version); }); }); From 1d2c427f961d6905f5df390e1ec5506d208e5811 Mon Sep 17 00:00:00 2001 From: Adir111 Date: Mon, 11 Nov 2024 12:03:38 +0200 Subject: [PATCH 13/14] removed .only --- core/api-server/tests/pipelines-versions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/api-server/tests/pipelines-versions.js b/core/api-server/tests/pipelines-versions.js index a25cde928..e8f479fe9 100644 --- a/core/api-server/tests/pipelines-versions.js +++ b/core/api-server/tests/pipelines-versions.js @@ -55,7 +55,7 @@ describe('Versions/Pipelines', () => { expect(specificVersion).to.eql(version); }); - it.only('should succeed to get versions and change version to latest', async () => { + it('should succeed to get versions and change version to latest', async () => { const { name, version: oldVersion } = await addPipeline(pipeline); const pipeline2 = clone(pipeline); pipeline2.options.ttl = 6666; From e466909bdd70aff804c44af42150a4a03b75d9a3 Mon Sep 17 00:00:00 2001 From: Adir111 Date: Mon, 11 Nov 2024 12:07:54 +0200 Subject: [PATCH 14/14] added validation for semver increasement of latest version --- core/api-server/tests/pipelines-versions.js | 1 + 1 file changed, 1 insertion(+) diff --git a/core/api-server/tests/pipelines-versions.js b/core/api-server/tests/pipelines-versions.js index e8f479fe9..0e2437ace 100644 --- a/core/api-server/tests/pipelines-versions.js +++ b/core/api-server/tests/pipelines-versions.js @@ -66,6 +66,7 @@ describe('Versions/Pipelines', () => { expect(semver).to.eql(['1.0.1', '1.0.0']); expect(oldVersion).to.be.not.equal(newVersion); expect(newVersion).to.be.equal(versionsList[0].version); + expect(versionsList[0].semver).to.be.equal('1.0.1'); }); });