From 24ec419b5d2a73d8fcd2b4988736eda68a531e9b Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Tue, 10 Dec 2024 02:29:05 +0100 Subject: [PATCH 01/13] wip --- examples/demo.js | 92 +++++---- managers/asset-operations-manager.js | 285 +++++++++++++++++++++------ 2 files changed, 275 insertions(+), 102 deletions(-) diff --git a/examples/demo.js b/examples/demo.js index fb17511..9cf46cb 100644 --- a/examples/demo.js +++ b/examples/demo.js @@ -28,51 +28,59 @@ function divider() { } (async () => { - // const content = { - // public: ` . - // "TL" .`, - // private: ` . - // "OT" . - // . - // . - - // `, - // }; const content = { - public: { - '@context': ['https://schema.org'], - '@id': 'uuid:1', - company: 'OT', - user: { - '@id': 'uuid:user:1', - }, - city: { - '@id': 'uuid:belgrade', - }, - }, - private: { - '@context': ['https://schema.org'], - '@graph': [ - { - '@id': 'uuid:user:1', - name: 'Adam', - lastname: 'Smith', - }, - { - '@id': 'uuid:belgrade', - title: 'Belgrade', - postCode: '11000', - }, - { - problem: 'empty', - }, - { - solution: 'generate', - }, - ], - }, + public: ` . + . + "TL" . + "TraceLabs" .`, + private: ` . + "OT" . + . + .`, }; + for (let i = 0; i < 1_000_000; i += 1) { + const id = Math.ceil(Math.random() * 1_000_000_000); + const index = id % 2; + content[ + index ? 'public' : 'private' + ] += `\n "object" .`; + } + // const content = { + // public: { + // '@context': ['https://schema.org'], + // '@id': 'uuid:1', + // company: 'OT', + // user: { + // '@id': 'uuid:user:1', + // }, + // city: { + // '@id': 'uuid:belgrade', + // }, + // }, + // private: { + // '@context': ['https://schema.org'], + // '@graph': [ + // { + // '@id': 'uuid:user:1', + // name: 'Adam', + // lastname: 'Smith', + // }, + // { + // '@id': 'uuid:belgrade', + // title: 'Belgrade', + // postCode: '11000', + // }, + // { + // problem: 'empty', + // }, + // { + // solution: 'generate', + // }, + // ], + // }, + // }; + // divider(); const nodeInfo = await DkgClient.node.info(); diff --git a/managers/asset-operations-manager.js b/managers/asset-operations-manager.js index 0a699dc..1b690dd 100644 --- a/managers/asset-operations-manager.js +++ b/managers/asset-operations-manager.js @@ -261,6 +261,54 @@ export default class AssetOperationsManager { .filter((line) => line !== ''); } + insertTripleSorted(triplesArray, newTriple) { + // Assuming triplesArray is already sorted + let left = 0; + let right = triplesArray.length; + while (left < right) { + const mid = Math.floor((left + right) / 2); + if (triplesArray[mid].localeCompare(newTriple) < 0) { + left = mid + 1; + } else { + right = mid; + } + } + triplesArray.splice(left, 0, newTriple); + } + splitConnectedArrays(publicTriples) { + const groupedPublic = []; + let currentSubject = publicTriples[0].split(' ')[0]; + let currentSubjectHash = currentSubject.startsWith('`; + let currentKA = [publicTriples[0]]; + + for (let i = 1; i < publicTriples.length; i++) { + const subject = publicTriples[i].split(' ')[0]; + const subjectHash = subject.startsWith('`; + + if ( + currentSubject === subject || + currentSubjectHash === subject || + subjectHash === currentSubject + ) { + currentKA.push(publicTriples[i]); + } else { + groupedPublic.push(currentKA); + currentSubject = subject; + currentSubjectHash = subjectHash; + currentKA = [publicTriples[i]]; + } + } + + // Push the last group + groupedPublic.push(currentKA); + + return groupedPublic; + } + /** * Creates a new knowledge collection. * @async @@ -322,86 +370,203 @@ export default class AssetOperationsManager { } else { dataset = await kcTools.formatDataset(content); } - - dataset.public = kcTools.generateMissingIdsForBlankNodes(dataset.public); + console.time('parsingAndSorting'); + /*dataset.public = kcTools.generateMissingIdsForBlankNodes(dataset.public); if (dataset.private) { dataset.private = kcTools.generateMissingIdsForBlankNodes(dataset.private); } - - // Ensure we have a list for private Tripless even if not provided - const publicTriplesSortedAndGrouped = kcTools.groupNquadsBySubject(dataset.public, true); - const privateTriplesSortedAndGrouped = dataset.private - ? kcTools.groupNquadsBySubject(dataset.private, true) - : []; - - // A helper function to generate the private merkle root Triples - function createPrivateMerkleRootTriple(subject, Tripless) { - const merkleRoot = kcTools.calculateMerkleRoot(Tripless); - return `${subject} <${PRIVATE_ASSERTION_PREDICATE}> "${merkleRoot}" .`; + let publicTriplesSortedAndGrouped = []; + if (dataset.private?.length) { + const privateTriplesSortedAndGrouped = dataset.private + ? kcTools.groupNquadsBySubject(dataset.private, true) + : []; + console.time('privateRoot'); + const privateRoot = kcTools.calculateMerkleRoot(privateTriplesSortedAndGrouped.flat()); + console.timeEnd('privateRoot'); + dataset.public.push([`_:b0 <${PRIVATE_ASSERTION_PREDICATE}> "${privateRoot}" .`]); + publicTriplesSortedAndGrouped = kcTools.groupNquadsBySubject(dataset.public, true); + if (privateTriplesSortedAndGrouped.length) { + let publicIndex = 0; + let privateIndex = 0; + + while ( + publicIndex < publicTriplesSortedAndGrouped.length || + privateIndex < privateTriplesSortedAndGrouped.length + ) { + // If we've exhausted all private Triples, we're done. + if (privateIndex === privateTriplesSortedAndGrouped.length) { + break; + } + + // If we've exhausted all public Triples, just insert in sorted manner remaining private Triples. + if (publicIndex === publicTriplesSortedAndGrouped.length) { + const [privateSubject] = + privateTriplesSortedAndGrouped[privateIndex][0].split(' '); + this.insertSortedOuterArray(publicTriplesSortedAndGrouped, [ + `<${ethers.sha256( + ethers.toUtf8Bytes(privateSubject), + )}> _:b0 .`, + ]); + privateIndex += 1; + continue; + } + // It this is slow we can check which index changed and just parse them + const [publicSubject] = + publicTriplesSortedAndGrouped[publicIndex][0].split(' '); + const [privateSubject] = + privateTriplesSortedAndGrouped[privateIndex][0].split(' '); + const compare = publicSubject.localeCompare(privateSubject); + + if (compare < 0) { + // The public subject comes before the private one, move forward in public + publicIndex += 1; + } else if (compare > 0) { + // The private subject comes before the public one, insert new Triples array in public + publicTriplesSortedAndGrouped.unshift([ + `<${ethers.sha256( + ethers.toUtf8Bytes(privateSubject), + )}> _:b0 .`, + ]); + publicIndex += 1; + privateIndex += 1; + } else { + // Subjects match, append private merkle root to the existing public Triples array in sorted manner + this.insertTripleSorted( + publicTriplesSortedAndGrouped[publicIndex], + `<${ethers.sha256( + ethers.toUtf8Bytes(privateSubject), + )}> _:b0 .`, + ); + publicIndex += 1; + privateIndex += 1; + } + } + } } - let publicIndex = 0; - let privateIndex = 0; + dataset.public = publicTriplesSortedAndGrouped.length + ? publicTriplesSortedAndGrouped.flat() + : kcTools.groupNquadsBySubject(dataset.public, true); +*/ + dataset.public = kcTools.generateMissingIdsForBlankNodes(dataset.public); + if (dataset.private) { + dataset.private = kcTools.generateMissingIdsForBlankNodes(dataset.private); + } - while ( - publicIndex < publicTriplesSortedAndGrouped.length || - privateIndex < privateTriplesSortedAndGrouped.length - ) { - // If we've exhausted all private Tripless, we're done. - if (privateIndex === privateTriplesSortedAndGrouped.length) { - break; + if (dataset.private?.length) { + const privateTriplesGrouped = kcTools.groupNquadsBySubject(dataset.private, true); + + console.time('privateRoot'); + // const privateRoot = kcTools.calculateMerkleRoot(privateTriplesGrouped.flat()); + console.timeEnd('privateRoot'); + + // dataset.public.push(`_:b0 <${PRIVATE_ASSERTION_PREDICATE}> "${privateRoot}" .`); + dataset.public.push(`_:b0 <${PRIVATE_ASSERTION_PREDICATE}> "A" .`); + + let publicTriplesGrouped = kcTools.groupNquadsBySubject(dataset.public, true); + + const mergedTriples = []; + + // Precompute hashed private subjects to avoid redundant computations + + // const [privateSubject] = group[0].split(' '); + // return { + // original: privateSubject, + // hashed: ``, + // }; + // }); + + // Create iterators for public and private groups + let publicIndex = 0; + let privateIndex = 0; + let tokenId = 1; + const publicLength = publicTriplesGrouped.length; + const privateLength = privateTriplesGrouped.length; + + const test1 = privateTriplesGrouped[privateIndex][0].split(' ')[0]; + const test2 = privateTriplesGrouped[privateIndex][0].split(' ')[0] + tokenId.toString(); + console.log(test1); + console.log(ethers.toUtf8Bytes(test1)); + console.log(test2); + console.log(ethers.toUtf8Bytes(test2)); + + // Merge public and private hashes triples in a single pass + console.time('merge'); + while (publicIndex < publicLength && privateIndex < privateLength) { + const publicGroup = publicTriplesGrouped[publicIndex]; + const [publicSubject] = publicGroup[0].split(' '); + const [privateSubject] = privateTriplesGrouped[privateIndex][0].split(' '); + + const compare = publicSubject.localeCompare(privateSubject); + if (compare < 0) { + // Public subject comes before private subject + mergedTriples.push(publicGroup); + publicIndex++; + tokenId++; + } else if (compare > 0) { + // Private subject comes before public subject + mergedTriples.push([ + `${``} _:b0 .`, + ]); + tokenId++; + privateIndex++; + } else { + // Subjects match, merge triples + this.insertTripleSorted( + publicGroup, + `${``} _:b0 .`, + ); + mergedTriples.push(publicGroup); + publicIndex++; + privateIndex++; + tokenId++; + } } - // If we've exhausted all public Tripless, just append all remaining private Tripless. - if (publicIndex === publicTriplesSortedAndGrouped.length) { - const [privateSubject] = privateTriplesSortedAndGrouped[privateIndex][0].split(' '); - const privateMerkleRootTriple = createPrivateMerkleRootTriple( - privateSubject, - privateTriplesSortedAndGrouped[privateIndex], - ); - publicTriplesSortedAndGrouped.push([privateMerkleRootTriple]); - privateIndex += 1; - continue; + // Append any remaining public triples + while (publicIndex < publicLength) { + mergedTriples.push(...publicTriplesGrouped[publicIndex]); + publicIndex++; } - const [publicSubject] = publicTriplesSortedAndGrouped[publicIndex][0].split(' '); - const [privateSubject] = privateTriplesSortedAndGrouped[privateIndex][0].split(' '); - const compare = publicSubject.localeCompare(privateSubject); - - if (compare < 0) { - // The public subject comes before the private one, move forward in public - publicIndex += 1; - } else if (compare > 0) { - // The private subject comes before the public one, insert new Tripless array in public - const privateMerkleRootTriple = createPrivateMerkleRootTriple( - privateSubject, - privateTriplesSortedAndGrouped[privateIndex], + // Append any remaining private triples + while (privateIndex < privateLength) { + mergedTriples.push( + `${``} _:b0 .`, ); - publicTriplesSortedAndGrouped.splice(publicIndex, 0, [privateMerkleRootTriple]); - publicIndex += 1; - privateIndex += 1; - } else { - // Subjects match, append private merkle root to the existing public Triples array - const privateMerkleRootTriple = createPrivateMerkleRootTriple( - privateSubject, - privateTriplesSortedAndGrouped[privateIndex], - ); - publicTriplesSortedAndGrouped[publicIndex].push(privateMerkleRootTriple); - publicIndex += 1; - privateIndex += 1; + privateIndex++; + tokenId++; } - } + console.timeEnd('merge'); - dataset.public = publicTriplesSortedAndGrouped.flat(); - if (dataset.private) { - dataset.private = privateTriplesSortedAndGrouped.flat(); + // Update the public dataset with the merged triples + dataset.public = mergedTriples.flat(); + } else { + // If there's no private dataset, ensure public is grouped correctly + dataset.public = kcTools.groupNquadsBySubject(dataset.public, true).flat(); } + console.timeEnd('parsingAndSorting'); + const numberOfChunks = kcTools.calculateNumberOfChunks(dataset.public, CHUNK_BYTE_SIZE); const datasetSize = numberOfChunks * CHUNK_BYTE_SIZE; this.validationService.validateAssertionSizeInBytes(datasetSize); + console.time('publicRoot'); const datasetRoot = kcTools.calculateMerkleRoot(dataset.public); + console.timeEnd('publicRoot'); + + console.time('regrout'); + const regroup = this.splitConnectedArrays(dataset.public); + console.timeEnd('regrout'); const contentAssetStorageAddress = await this.blockchainService.getContractAddress( 'ContentAssetStorage', From 35c3aac5e917b9085902088c397f2447a5851e1c Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Tue, 10 Dec 2024 11:44:57 +0100 Subject: [PATCH 02/13] rework private --- examples/demo.js | 100 ++++++++-------- managers/asset-operations-manager.js | 163 ++------------------------- 2 files changed, 62 insertions(+), 201 deletions(-) diff --git a/examples/demo.js b/examples/demo.js index 9cf46cb..d6136e9 100644 --- a/examples/demo.js +++ b/examples/demo.js @@ -28,59 +28,59 @@ function divider() { } (async () => { - const content = { - public: ` . - . - "TL" . - "TraceLabs" .`, - private: ` . - "OT" . - . - .`, - }; - - for (let i = 0; i < 1_000_000; i += 1) { - const id = Math.ceil(Math.random() * 1_000_000_000); - const index = id % 2; - content[ - index ? 'public' : 'private' - ] += `\n "object" .`; - } // const content = { - // public: { - // '@context': ['https://schema.org'], - // '@id': 'uuid:1', - // company: 'OT', - // user: { - // '@id': 'uuid:user:1', - // }, - // city: { - // '@id': 'uuid:belgrade', - // }, - // }, - // private: { - // '@context': ['https://schema.org'], - // '@graph': [ - // { - // '@id': 'uuid:user:1', - // name: 'Adam', - // lastname: 'Smith', - // }, - // { - // '@id': 'uuid:belgrade', - // title: 'Belgrade', - // postCode: '11000', - // }, - // { - // problem: 'empty', - // }, - // { - // solution: 'generate', - // }, - // ], - // }, + // // public: ` . + // // . + // // "TL" . + // // "TraceLabs" .`, + // private: ` . + // "OT" . + // . + // .`, // }; + // for (let i = 0; i < 1_000_000; i += 1) { + // const id = Math.ceil(Math.random() * 100_000); + // const index = id % 2; + // content[ + // index ? 'public' : 'private' + // ] += `\n "object" .`; + // } + const content = { + public: { + '@context': ['https://schema.org'], + '@id': 'uuid:1', + company: 'OT', + user: { + '@id': 'uuid:user:1', + }, + city: { + '@id': 'uuid:belgrade', + }, + }, + // private: { + // '@context': ['https://schema.org'], + // '@graph': [ + // { + // '@id': 'uuid:user:1', + // name: 'Adam', + // lastname: 'Smith', + // }, + // { + // '@id': 'uuid:belgrade', + // title: 'Belgrade', + // postCode: '11000', + // }, + // { + // problem: 'empty', + // }, + // { + // solution: 'generate', + // }, + // ], + // }, + }; + // divider(); const nodeInfo = await DkgClient.node.info(); diff --git a/managers/asset-operations-manager.js b/managers/asset-operations-manager.js index 1b690dd..9e48f1e 100644 --- a/managers/asset-operations-manager.js +++ b/managers/asset-operations-manager.js @@ -275,39 +275,6 @@ export default class AssetOperationsManager { } triplesArray.splice(left, 0, newTriple); } - splitConnectedArrays(publicTriples) { - const groupedPublic = []; - let currentSubject = publicTriples[0].split(' ')[0]; - let currentSubjectHash = currentSubject.startsWith('`; - let currentKA = [publicTriples[0]]; - - for (let i = 1; i < publicTriples.length; i++) { - const subject = publicTriples[i].split(' ')[0]; - const subjectHash = subject.startsWith('`; - - if ( - currentSubject === subject || - currentSubjectHash === subject || - subjectHash === currentSubject - ) { - currentKA.push(publicTriples[i]); - } else { - groupedPublic.push(currentKA); - currentSubject = subject; - currentSubjectHash = subjectHash; - currentKA = [publicTriples[i]]; - } - } - - // Push the last group - groupedPublic.push(currentKA); - - return groupedPublic; - } /** * Creates a new knowledge collection. @@ -363,6 +330,8 @@ export default class AssetOperationsManager { ) { if (content.public) { dataset.public = this.processContent(content.public); + } else { + dataset.public = []; } if (content.private && typeof content.private === 'string') { dataset.private = this.processContent(content.private); @@ -370,128 +339,26 @@ export default class AssetOperationsManager { } else { dataset = await kcTools.formatDataset(content); } - console.time('parsingAndSorting'); - /*dataset.public = kcTools.generateMissingIdsForBlankNodes(dataset.public); - if (dataset.private) { - dataset.private = kcTools.generateMissingIdsForBlankNodes(dataset.private); - } - let publicTriplesSortedAndGrouped = []; - if (dataset.private?.length) { - const privateTriplesSortedAndGrouped = dataset.private - ? kcTools.groupNquadsBySubject(dataset.private, true) - : []; - console.time('privateRoot'); - const privateRoot = kcTools.calculateMerkleRoot(privateTriplesSortedAndGrouped.flat()); - console.timeEnd('privateRoot'); - dataset.public.push([`_:b0 <${PRIVATE_ASSERTION_PREDICATE}> "${privateRoot}" .`]); - publicTriplesSortedAndGrouped = kcTools.groupNquadsBySubject(dataset.public, true); - if (privateTriplesSortedAndGrouped.length) { - let publicIndex = 0; - let privateIndex = 0; - - while ( - publicIndex < publicTriplesSortedAndGrouped.length || - privateIndex < privateTriplesSortedAndGrouped.length - ) { - // If we've exhausted all private Triples, we're done. - if (privateIndex === privateTriplesSortedAndGrouped.length) { - break; - } - - // If we've exhausted all public Triples, just insert in sorted manner remaining private Triples. - if (publicIndex === publicTriplesSortedAndGrouped.length) { - const [privateSubject] = - privateTriplesSortedAndGrouped[privateIndex][0].split(' '); - this.insertSortedOuterArray(publicTriplesSortedAndGrouped, [ - `<${ethers.sha256( - ethers.toUtf8Bytes(privateSubject), - )}> _:b0 .`, - ]); - privateIndex += 1; - continue; - } - // It this is slow we can check which index changed and just parse them - const [publicSubject] = - publicTriplesSortedAndGrouped[publicIndex][0].split(' '); - const [privateSubject] = - privateTriplesSortedAndGrouped[privateIndex][0].split(' '); - const compare = publicSubject.localeCompare(privateSubject); - - if (compare < 0) { - // The public subject comes before the private one, move forward in public - publicIndex += 1; - } else if (compare > 0) { - // The private subject comes before the public one, insert new Triples array in public - publicTriplesSortedAndGrouped.unshift([ - `<${ethers.sha256( - ethers.toUtf8Bytes(privateSubject), - )}> _:b0 .`, - ]); - publicIndex += 1; - privateIndex += 1; - } else { - // Subjects match, append private merkle root to the existing public Triples array in sorted manner - this.insertTripleSorted( - publicTriplesSortedAndGrouped[publicIndex], - `<${ethers.sha256( - ethers.toUtf8Bytes(privateSubject), - )}> _:b0 .`, - ); - publicIndex += 1; - privateIndex += 1; - } - } - } - } - - dataset.public = publicTriplesSortedAndGrouped.length - ? publicTriplesSortedAndGrouped.flat() - : kcTools.groupNquadsBySubject(dataset.public, true); -*/ - dataset.public = kcTools.generateMissingIdsForBlankNodes(dataset.public); - if (dataset.private) { - dataset.private = kcTools.generateMissingIdsForBlankNodes(dataset.private); - } + if (dataset.public.length) + dataset.public = kcTools.generateMissingIdsForBlankNodes(dataset.public); if (dataset.private?.length) { + dataset.private = kcTools.generateMissingIdsForBlankNodes(dataset.private); const privateTriplesGrouped = kcTools.groupNquadsBySubject(dataset.private, true); - console.time('privateRoot'); - // const privateRoot = kcTools.calculateMerkleRoot(privateTriplesGrouped.flat()); - console.timeEnd('privateRoot'); + const privateRoot = kcTools.calculateMerkleRoot(privateTriplesGrouped.flat()); - // dataset.public.push(`_:b0 <${PRIVATE_ASSERTION_PREDICATE}> "${privateRoot}" .`); - dataset.public.push(`_:b0 <${PRIVATE_ASSERTION_PREDICATE}> "A" .`); + dataset.public.push(`_:b0 <${PRIVATE_ASSERTION_PREDICATE}> "${privateRoot}" .`); let publicTriplesGrouped = kcTools.groupNquadsBySubject(dataset.public, true); const mergedTriples = []; - - // Precompute hashed private subjects to avoid redundant computations - - // const [privateSubject] = group[0].split(' '); - // return { - // original: privateSubject, - // hashed: ``, - // }; - // }); - - // Create iterators for public and private groups let publicIndex = 0; let privateIndex = 0; let tokenId = 1; const publicLength = publicTriplesGrouped.length; const privateLength = privateTriplesGrouped.length; - const test1 = privateTriplesGrouped[privateIndex][0].split(' ')[0]; - const test2 = privateTriplesGrouped[privateIndex][0].split(' ')[0] + tokenId.toString(); - console.log(test1); - console.log(ethers.toUtf8Bytes(test1)); - console.log(test2); - console.log(ethers.toUtf8Bytes(test2)); - // Merge public and private hashes triples in a single pass console.time('merge'); while (publicIndex < publicLength && privateIndex < privateLength) { @@ -509,7 +376,7 @@ export default class AssetOperationsManager { // Private subject comes before public subject mergedTriples.push([ `${``} _:b0 .`, ]); tokenId++; @@ -519,7 +386,7 @@ export default class AssetOperationsManager { this.insertTripleSorted( publicGroup, `${``} _:b0 .`, ); mergedTriples.push(publicGroup); @@ -537,9 +404,10 @@ export default class AssetOperationsManager { // Append any remaining private triples while (privateIndex < privateLength) { + const [privateSubject] = privateTriplesGrouped[privateIndex][0].split(' '); mergedTriples.push( `${``} _:b0 .`, ); privateIndex++; @@ -554,19 +422,12 @@ export default class AssetOperationsManager { dataset.public = kcTools.groupNquadsBySubject(dataset.public, true).flat(); } - console.timeEnd('parsingAndSorting'); - const numberOfChunks = kcTools.calculateNumberOfChunks(dataset.public, CHUNK_BYTE_SIZE); const datasetSize = numberOfChunks * CHUNK_BYTE_SIZE; this.validationService.validateAssertionSizeInBytes(datasetSize); - console.time('publicRoot'); - const datasetRoot = kcTools.calculateMerkleRoot(dataset.public); - console.timeEnd('publicRoot'); - console.time('regrout'); - const regroup = this.splitConnectedArrays(dataset.public); - console.timeEnd('regrout'); + const datasetRoot = kcTools.calculateMerkleRoot(dataset.public); const contentAssetStorageAddress = await this.blockchainService.getContractAddress( 'ContentAssetStorage', From a7e32b1da4866b2ef0966fe795193844de3dff16 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Tue, 10 Dec 2024 12:09:55 +0100 Subject: [PATCH 03/13] tiding up --- constants.js | 5 +++- examples/demo.js | 42 ++++++++++++++-------------- managers/asset-operations-manager.js | 38 ++++++++++++------------- 3 files changed, 43 insertions(+), 42 deletions(-) diff --git a/constants.js b/constants.js index 87d71a7..b67e12a 100644 --- a/constants.js +++ b/constants.js @@ -11,7 +11,10 @@ export const MAX_FILE_SIZE = 524288000; export const DID_PREFIX = 'did:dkg'; export const PRIVATE_ASSERTION_PREDICATE = - 'https://ontology.origintrail.io/dkg/1.0#privateAssertionID'; + 'https://ontology.origintrail.io/dkg/1.0#privateMerkleRoot'; + +export const PRIVATE_RESOURCE_PREDICATE = + 'https://ontology.origintrail.io/dkg/1.0#representsPrivateResource'; export const ZERO_ADDRESS = '0x0000000000000000000000000000000000000000'; diff --git a/examples/demo.js b/examples/demo.js index d6136e9..c8d4d61 100644 --- a/examples/demo.js +++ b/examples/demo.js @@ -58,27 +58,27 @@ function divider() { '@id': 'uuid:belgrade', }, }, - // private: { - // '@context': ['https://schema.org'], - // '@graph': [ - // { - // '@id': 'uuid:user:1', - // name: 'Adam', - // lastname: 'Smith', - // }, - // { - // '@id': 'uuid:belgrade', - // title: 'Belgrade', - // postCode: '11000', - // }, - // { - // problem: 'empty', - // }, - // { - // solution: 'generate', - // }, - // ], - // }, + private: { + '@context': ['https://schema.org'], + '@graph': [ + { + '@id': 'uuid:user:1', + name: 'Adam', + lastname: 'Smith', + }, + { + '@id': 'uuid:belgrade', + title: 'Belgrade', + postCode: '11000', + }, + { + problem: 'empty', + }, + { + solution: 'generate', + }, + ], + }, }; // divider(); diff --git a/managers/asset-operations-manager.js b/managers/asset-operations-manager.js index 9e48f1e..e67bfb2 100644 --- a/managers/asset-operations-manager.js +++ b/managers/asset-operations-manager.js @@ -13,6 +13,7 @@ import { CHUNK_BYTE_SIZE, OPERATION_DELAYS, PRIVATE_ASSERTION_PREDICATE, + PRIVATE_RESOURCE_PREDICATE, } from '../constants.js'; import emptyHooks from '../util/empty-hooks.js'; @@ -339,8 +340,6 @@ export default class AssetOperationsManager { } else { dataset = await kcTools.formatDataset(content); } - if (dataset.public.length) - dataset.public = kcTools.generateMissingIdsForBlankNodes(dataset.public); if (dataset.private?.length) { dataset.private = kcTools.generateMissingIdsForBlankNodes(dataset.private); @@ -350,12 +349,14 @@ export default class AssetOperationsManager { dataset.public.push(`_:b0 <${PRIVATE_ASSERTION_PREDICATE}> "${privateRoot}" .`); + if (dataset.public.length) { + dataset.public = kcTools.generateMissingIdsForBlankNodes(dataset.public); + } let publicTriplesGrouped = kcTools.groupNquadsBySubject(dataset.public, true); const mergedTriples = []; let publicIndex = 0; let privateIndex = 0; - let tokenId = 1; const publicLength = publicTriplesGrouped.length; const privateLength = privateTriplesGrouped.length; @@ -371,28 +372,27 @@ export default class AssetOperationsManager { // Public subject comes before private subject mergedTriples.push(publicGroup); publicIndex++; - tokenId++; } else if (compare > 0) { // Private subject comes before public subject mergedTriples.push([ - `${``} _:b0 .`, + `${``} <${PRIVATE_RESOURCE_PREDICATE}> _:b0 .`, ]); - tokenId++; privateIndex++; } else { // Subjects match, merge triples this.insertTripleSorted( publicGroup, - `${``} _:b0 .`, + `${``} <${PRIVATE_RESOURCE_PREDICATE}> _:b0 .`, ); mergedTriples.push(publicGroup); publicIndex++; privateIndex++; - tokenId++; } } @@ -405,16 +405,14 @@ export default class AssetOperationsManager { // Append any remaining private triples while (privateIndex < privateLength) { const [privateSubject] = privateTriplesGrouped[privateIndex][0].split(' '); - mergedTriples.push( - `${``} _:b0 .`, - ); + mergedTriples.push([ + `${``} <${PRIVATE_RESOURCE_PREDICATE}> _:b0 .`, + ]); privateIndex++; - tokenId++; } - console.timeEnd('merge'); - // Update the public dataset with the merged triples dataset.public = mergedTriples.flat(); } else { From 2795e8ba65300384f7605e246dde76f64806cba1 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Tue, 10 Dec 2024 12:11:24 +0100 Subject: [PATCH 04/13] remove debug console --- managers/asset-operations-manager.js | 1 - 1 file changed, 1 deletion(-) diff --git a/managers/asset-operations-manager.js b/managers/asset-operations-manager.js index e67bfb2..22955cb 100644 --- a/managers/asset-operations-manager.js +++ b/managers/asset-operations-manager.js @@ -361,7 +361,6 @@ export default class AssetOperationsManager { const privateLength = privateTriplesGrouped.length; // Merge public and private hashes triples in a single pass - console.time('merge'); while (publicIndex < publicLength && privateIndex < privateLength) { const publicGroup = publicTriplesGrouped[publicIndex]; const [publicSubject] = publicGroup[0].split(' '); From 1aa23e0e7b9afb42e10f0a24d0e5cd4d7364b344 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Tue, 10 Dec 2024 16:06:20 +0100 Subject: [PATCH 05/13] Add hash prefix constant --- constants.js | 2 ++ managers/asset-operations-manager.js | 31 +++++++++++++--------------- package-lock.json | 14 ++++++------- package.json | 2 +- 4 files changed, 24 insertions(+), 25 deletions(-) diff --git a/constants.js b/constants.js index b67e12a..f19c4e4 100644 --- a/constants.js +++ b/constants.js @@ -16,6 +16,8 @@ export const PRIVATE_ASSERTION_PREDICATE = export const PRIVATE_RESOURCE_PREDICATE = 'https://ontology.origintrail.io/dkg/1.0#representsPrivateResource'; +export const PRIVATE_HASH_SUBJECT_PREFIX = 'https://ontology.origintrail.io/dkg/1.0#metadata-hash:'; + export const ZERO_ADDRESS = '0x0000000000000000000000000000000000000000'; export const LABEL_PREFIX = ''; diff --git a/managers/asset-operations-manager.js b/managers/asset-operations-manager.js index 22955cb..c71a777 100644 --- a/managers/asset-operations-manager.js +++ b/managers/asset-operations-manager.js @@ -14,6 +14,7 @@ import { OPERATION_DELAYS, PRIVATE_ASSERTION_PREDICATE, PRIVATE_RESOURCE_PREDICATE, + PRIVATE_HASH_SUBJECT_PREFIX, } from '../constants.js'; import emptyHooks from '../util/empty-hooks.js'; @@ -347,7 +348,9 @@ export default class AssetOperationsManager { const privateRoot = kcTools.calculateMerkleRoot(privateTriplesGrouped.flat()); - dataset.public.push(`_:b0 <${PRIVATE_ASSERTION_PREDICATE}> "${privateRoot}" .`); + dataset.public.push( + `<${kaTools.generateNamedNode()}> <${PRIVATE_ASSERTION_PREDICATE}> "${privateRoot}" .`, + ); if (dataset.public.length) { dataset.public = kcTools.generateMissingIdsForBlankNodes(dataset.public); @@ -373,21 +376,13 @@ export default class AssetOperationsManager { publicIndex++; } else if (compare > 0) { // Private subject comes before public subject - mergedTriples.push([ - `${``} <${PRIVATE_RESOURCE_PREDICATE}> _:b0 .`, - ]); + mergedTriples.push([this.generatePrivateRepresentation(privateSubject)]); privateIndex++; } else { // Subjects match, merge triples this.insertTripleSorted( publicGroup, - `${``} <${PRIVATE_RESOURCE_PREDICATE}> _:b0 .`, + this.generatePrivateRepresentation(privateSubject), ); mergedTriples.push(publicGroup); publicIndex++; @@ -404,12 +399,7 @@ export default class AssetOperationsManager { // Append any remaining private triples while (privateIndex < privateLength) { const [privateSubject] = privateTriplesGrouped[privateIndex][0].split(' '); - mergedTriples.push([ - `${``} <${PRIVATE_RESOURCE_PREDICATE}> _:b0 .`, - ]); + mergedTriples.push([this.generatePrivateRepresentation(privateSubject)]); privateIndex++; } // Update the public dataset with the merged triples @@ -564,6 +554,13 @@ export default class AssetOperationsManager { }; } + generatePrivateRepresentation(privateSubject) { + return `${`<${PRIVATE_HASH_SUBJECT_PREFIX}${ethers.solidityPackedSha256( + ['string'], + [privateSubject.slice(1, -1)], + )}>`} <${PRIVATE_RESOURCE_PREDICATE}> <${kaTools.generateNamedNode()}> .`; + } + /** * Transfer an asset to a new owner on a specified blockchain. * @async diff --git a/package-lock.json b/package-lock.json index ee38d18..8841f42 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "8.0.0-alpha.5", "license": "Apache-2.0", "dependencies": { - "assertion-tools": "^8.0.0-gamma.0", + "assertion-tools": "^8.0.0-gamma.1", "axios": "^0.27.2", "dkg-evm-module": "^8.0.2-alpha.0", "ethers": "^6.1.0", @@ -3109,9 +3109,9 @@ } }, "node_modules/assertion-tools": { - "version": "8.0.0-gamma.0", - "resolved": "https://registry.npmjs.org/assertion-tools/-/assertion-tools-8.0.0-gamma.0.tgz", - "integrity": "sha512-tUF2IcrddKhz2usRsgjrEGdfMuScz2P2nt8/9oJVb/NiSUp4l8i1nDDvmpAT3oompWLbd9HAxo2I0P4S4mJCYA==", + "version": "8.0.0-gamma.1", + "resolved": "https://registry.npmjs.org/assertion-tools/-/assertion-tools-8.0.0-gamma.1.tgz", + "integrity": "sha512-I4IHIiylrVFUoLy07qPxCbwwUk0Qpqfo0qL1cPBKcSY0e9b4+AHQoBLuSZkaE9rUy1wC+7vqBMikn4d/DW3FAw==", "dependencies": { "ethers": "^5.7.2", "jsonld": "^8.1.0", @@ -14213,9 +14213,9 @@ "integrity": "sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==" }, "assertion-tools": { - "version": "8.0.0-gamma.0", - "resolved": "https://registry.npmjs.org/assertion-tools/-/assertion-tools-8.0.0-gamma.0.tgz", - "integrity": "sha512-tUF2IcrddKhz2usRsgjrEGdfMuScz2P2nt8/9oJVb/NiSUp4l8i1nDDvmpAT3oompWLbd9HAxo2I0P4S4mJCYA==", + "version": "8.0.0-gamma.1", + "resolved": "https://registry.npmjs.org/assertion-tools/-/assertion-tools-8.0.0-gamma.1.tgz", + "integrity": "sha512-I4IHIiylrVFUoLy07qPxCbwwUk0Qpqfo0qL1cPBKcSY0e9b4+AHQoBLuSZkaE9rUy1wC+7vqBMikn4d/DW3FAw==", "requires": { "ethers": "^5.7.2", "jsonld": "^8.1.0", diff --git a/package.json b/package.json index 9a0e054..6363c5c 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ }, "homepage": "https://github.com/OriginTrail/dkg.js/tree/main#readme", "dependencies": { - "assertion-tools": "^8.0.0-gamma.0", + "assertion-tools": "^8.0.0-gamma.1", "axios": "^0.27.2", "dkg-evm-module": "^8.0.2-alpha.0", "ethers": "^6.1.0", From 7020a839da3fbf04a03a92ac70885f0d37a1d2ee Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Tue, 10 Dec 2024 16:25:24 +0100 Subject: [PATCH 06/13] fix todo comment --- managers/asset-operations-manager.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/managers/asset-operations-manager.js b/managers/asset-operations-manager.js index c71a777..f57d5af 100644 --- a/managers/asset-operations-manager.js +++ b/managers/asset-operations-manager.js @@ -527,7 +527,7 @@ export default class AssetOperationsManager { let finalityOperationResult = null; - // TO DO: ADD OPTIONAL WAITING FOR FINALITY + // TODO: ADD OPTIONAL WAITING FOR FINALITY try { finalityOperationResult = await this.nodeApiService.getOperationResult( endpoint, From 279d22d9d45fd76095c0a3fd2457398180fc67da Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Thu, 12 Dec 2024 12:29:18 +0100 Subject: [PATCH 07/13] data processing --- examples/demo.js | 9 -- managers/asset-operations-manager.js | 192 ++++++++++++++++++++++++++- 2 files changed, 188 insertions(+), 13 deletions(-) diff --git a/examples/demo.js b/examples/demo.js index c8d4d61..22bd0ec 100644 --- a/examples/demo.js +++ b/examples/demo.js @@ -97,13 +97,4 @@ function divider() { console.log(createAssetResult); divider(); - - const createCollectionResult = await DkgClient.graph.create(content, { - epochsNum: 2, - tokenAmount: '100', - }); - console.log('======================== ASSET CREATED'); - console.log(createCollectionResult); - - divider(); })(); diff --git a/managers/asset-operations-manager.js b/managers/asset-operations-manager.js index f57d5af..4fac995 100644 --- a/managers/asset-operations-manager.js +++ b/managers/asset-operations-manager.js @@ -553,11 +553,12 @@ export default class AssetOperationsManager { }, }; } - + hashSubject(subject) { + ethers.solidityPackedSha256(['string'], [subject.slice(1, -1)]); + } generatePrivateRepresentation(privateSubject) { - return `${`<${PRIVATE_HASH_SUBJECT_PREFIX}${ethers.solidityPackedSha256( - ['string'], - [privateSubject.slice(1, -1)], + return `${`<${PRIVATE_HASH_SUBJECT_PREFIX}${hashSubject( + privateSubject, )}>`} <${PRIVATE_RESOURCE_PREDICATE}> <${kaTools.generateNamedNode()}> .`; } @@ -984,6 +985,189 @@ export default class AssetOperationsManager { dataset = await kcTools.formatDataset(content); } + const getOperationId = await this.nodeApiService.get( + endpoint, + port, + authToken, + UAL, + '', + false, + true, + null, + 1, + null, + ); + + const subjectUALPairsResult = await this.nodeApiService.getOperationResult( + endpoint, + port, + authToken, + OPERATIONS.GET, + maxNumberOfRetries, + frequency, + getOperationId, + ); + const subjectUALMap = new Map(); + const subjectHashUALMap = new Map(); + + const { subjectUALPairs, privateMerkleRootTriple } = subjectUALPairsResult; + subjectUALPairs.forEach((pair) => { + if (pair.subject) { + subjectUALMap.set(pair.subject, pair.UAL); + } + if (pair.privateSubjectHash) { + subjectHashUALMap.set(pair.privateSubjectHash, pair.UAL); + } + }); + + if (dataset.private?.length) { + dataset.private = kcTools.generateMissingIdsForBlankNodes(dataset.private); + const privateTriplesGrouped = kcTools.groupNquadsBySubject(dataset.private, true); + + const privateRoot = kcTools.calculateMerkleRoot(privateTriplesGrouped.flat()); + + let publicTriplesGrouped = []; + if (dataset.public?.length) { + dataset.public = kcTools.generateMissingIdsForBlankNodes(dataset.public); + } + if (privateMerkleRootTriple) { + publicTriplesGrouped = kcTools.groupNquadsBySubject(dataset.public, true); + const [privateMerkleRootTripleSubject] = privateMerkleRootTriple.split(' '); + let privateMerkleRootTripleExists = false; + // This is sorted is should probably be done by binary search + for (const [index, [triple]] of publicTriplesGrouped.entries()) { + const [subject] = triple.split(' '); + if (privateMerkleRootTripleSubject === subject) { + const changePrivateRootIndex = publicTriplesGrouped[index].findIndex( + PRIVATE_RESOURCE_PREDICATE, + ); + publicTriplesGrouped[index][ + changePrivateRootIndex + ] = `${privateMerkleRootTripleSubject} <${PRIVATE_ASSERTION_PREDICATE}> "${privateRoot}" .`; + privateMerkleRootTripleExists = true; + } + } + } else { + dataset.public.push( + `<${kaTools.generateNamedNode()}> <${PRIVATE_ASSERTION_PREDICATE}> "${privateRoot}" .`, + ); + publicTriplesGrouped = kcTools.groupNquadsBySubject(dataset.public, true); + } + + const mergedTriples = []; + let publicIndex = 0; + let privateIndex = 0; + const publicLength = publicTriplesGrouped.length; + const privateLength = privateTriplesGrouped.length; + let tokensToBeMinted = 0; + // Merge public and private hashes triples in a single pass + while (publicIndex < publicLength && privateIndex < privateLength) { + const publicGroup = publicTriplesGrouped[publicIndex]; + const [publicSubject] = publicGroup[0].split(' '); + const [privateSubject] = privateTriplesGrouped[privateIndex][0].split(' '); + + // If this subject existed in previous state remove it from maps as it already has a token + const subjectExistedInPreviousState = + subjectUALMap.delete(publicSubject) || + subjectHashUALMap.delete(this.hashSubject(publicSubject)); + // If it didn't exist than we need to mint new asset + if (!subjectExistedInPreviousState) { + tokensToBeMinted += 1; + } + const compare = publicSubject.localeCompare(privateSubject); + if (compare < 0) { + // If this subject existed in previous state remove it from maps as it already has a token + const subjectExistedInPreviousState = + subjectUALMap.delete(publicSubject) || + subjectHashUALMap.delete(this.hashSubject(publicSubject)); + // If it didn't exist than we need to mint new token + if (!subjectExistedInPreviousState) { + tokensToBeMinted += 1; + } + // Public subject comes before private subject + mergedTriples.push(publicGroup); + publicIndex++; + } else if (compare > 0) { + // Private subject comes before public subject + // If this subject existed in previous state remove it from maps as it already has a token + const subjectExistedInPreviousState = + subjectUALMap.delete(privateSubject) || + subjectHashUALMap.delete(this.hashSubject(privateSubject)); + // If it didn't exist than we need to mint new token + if (!subjectExistedInPreviousState) { + tokensToBeMinted += 1; + } + mergedTriples.push([this.generatePrivateRepresentation(privateSubject)]); + privateIndex++; + } else { + // Subjects match, merge triples + // If this subject existed in previous state remove it from maps as it already has a token + const subjectExistedInPreviousState = + subjectUALMap.delete(privateSubject) || + subjectHashUALMap.delete(this.hashSubject(privateSubject)); + // If it didn't exist than we need to mint new token + if (!subjectExistedInPreviousState) { + tokensToBeMinted += 1; + } + this.insertTripleSorted( + publicGroup, + this.generatePrivateRepresentation(privateSubject), + ); + mergedTriples.push(publicGroup); + publicIndex++; + privateIndex++; + } + } + + // Append any remaining public triples + while (publicIndex < publicLength) { + const [publicSubject] = publicGroup[0].split(' '); + const subjectExistedInPreviousState = + subjectUALMap.delete(publicSubject) || + subjectHashUALMap.delete(this.hashSubject(publicSubject)); + // If it didn't exist than we need to mint new token + if (!subjectExistedInPreviousState) { + tokensToBeMinted += 1; + } + mergedTriples.push(...publicTriplesGrouped[publicIndex]); + publicIndex++; + } + + // Append any remaining private triples + while (privateIndex < privateLength) { + const [privateSubject] = privateTriplesGrouped[privateIndex][0].split(' '); + const subjectExistedInPreviousState = + subjectUALMap.delete(privateSubject) || + subjectHashUALMap.delete(this.hashSubject(privateSubject)); + // If it didn't exist than we need to mint new token + if (!subjectExistedInPreviousState) { + tokensToBeMinted += 1; + } + mergedTriples.push([this.generatePrivateRepresentation(privateSubject)]); + privateIndex++; + } + // Update the public dataset with the merged triples + dataset.public = mergedTriples.flat(); + } else { + // If there's no private dataset, ensure public is grouped correctly + dataset.public = kcTools.groupNquadsBySubject(dataset.public, true).flat(); + + for (const [publicTriple] of dataset.public) { + const [publicSubject] = publicTriple[0].split(' '); + const subjectExistedInPreviousState = + subjectUALMap.delete(publicSubject) || + subjectHashUALMap.delete(this.hashSubject(publicSubject)); + if (!subjectExistedInPreviousState) { + tokensToBeMinted += 1; + } + } + } + + // Find all old subject UALs that are not in new dataset + const tokensToBeBurned = new Set([...subjectUALMap.values(), ...subjectHashUALMap.values()]) + .map((ual) => ual.split('/').pop()) + .sort(); + const numberOfChunks = kcTools.calculateNumberOfChunks(dataset, CHUNK_BYTE_SIZE); const datasetSize = numberOfChunks * CHUNK_BYTE_SIZE; From e9b9241fe6c15aed08c112cac619eafee43884b8 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Fri, 13 Dec 2024 18:29:03 +0100 Subject: [PATCH 08/13] cleanup --- managers/asset-operations-manager.js | 123 ++------------------------- 1 file changed, 9 insertions(+), 114 deletions(-) diff --git a/managers/asset-operations-manager.js b/managers/asset-operations-manager.js index 4fac995..9ce1f6d 100644 --- a/managers/asset-operations-manager.js +++ b/managers/asset-operations-manager.js @@ -1020,16 +1020,21 @@ export default class AssetOperationsManager { } }); + // Handle inserting private root triple in new public + // Private exist in new assertion if (dataset.private?.length) { dataset.private = kcTools.generateMissingIdsForBlankNodes(dataset.private); const privateTriplesGrouped = kcTools.groupNquadsBySubject(dataset.private, true); - - const privateRoot = kcTools.calculateMerkleRoot(privateTriplesGrouped.flat()); + dataset.private = privateTriplesGrouped.flat(); + const privateRoot = kcTools.calculateMerkleRoot( + privateTriplesGrouped.flat(dataset.private), + ); let publicTriplesGrouped = []; if (dataset.public?.length) { dataset.public = kcTools.generateMissingIdsForBlankNodes(dataset.public); } + // Private root triple existed before update it if (privateMerkleRootTriple) { publicTriplesGrouped = kcTools.groupNquadsBySubject(dataset.public, true); const [privateMerkleRootTripleSubject] = privateMerkleRootTriple.split(' '); @@ -1047,126 +1052,16 @@ export default class AssetOperationsManager { privateMerkleRootTripleExists = true; } } + // Private root triple didn't existed before insert it } else { dataset.public.push( `<${kaTools.generateNamedNode()}> <${PRIVATE_ASSERTION_PREDICATE}> "${privateRoot}" .`, ); publicTriplesGrouped = kcTools.groupNquadsBySubject(dataset.public, true); } - - const mergedTriples = []; - let publicIndex = 0; - let privateIndex = 0; - const publicLength = publicTriplesGrouped.length; - const privateLength = privateTriplesGrouped.length; - let tokensToBeMinted = 0; - // Merge public and private hashes triples in a single pass - while (publicIndex < publicLength && privateIndex < privateLength) { - const publicGroup = publicTriplesGrouped[publicIndex]; - const [publicSubject] = publicGroup[0].split(' '); - const [privateSubject] = privateTriplesGrouped[privateIndex][0].split(' '); - - // If this subject existed in previous state remove it from maps as it already has a token - const subjectExistedInPreviousState = - subjectUALMap.delete(publicSubject) || - subjectHashUALMap.delete(this.hashSubject(publicSubject)); - // If it didn't exist than we need to mint new asset - if (!subjectExistedInPreviousState) { - tokensToBeMinted += 1; - } - const compare = publicSubject.localeCompare(privateSubject); - if (compare < 0) { - // If this subject existed in previous state remove it from maps as it already has a token - const subjectExistedInPreviousState = - subjectUALMap.delete(publicSubject) || - subjectHashUALMap.delete(this.hashSubject(publicSubject)); - // If it didn't exist than we need to mint new token - if (!subjectExistedInPreviousState) { - tokensToBeMinted += 1; - } - // Public subject comes before private subject - mergedTriples.push(publicGroup); - publicIndex++; - } else if (compare > 0) { - // Private subject comes before public subject - // If this subject existed in previous state remove it from maps as it already has a token - const subjectExistedInPreviousState = - subjectUALMap.delete(privateSubject) || - subjectHashUALMap.delete(this.hashSubject(privateSubject)); - // If it didn't exist than we need to mint new token - if (!subjectExistedInPreviousState) { - tokensToBeMinted += 1; - } - mergedTriples.push([this.generatePrivateRepresentation(privateSubject)]); - privateIndex++; - } else { - // Subjects match, merge triples - // If this subject existed in previous state remove it from maps as it already has a token - const subjectExistedInPreviousState = - subjectUALMap.delete(privateSubject) || - subjectHashUALMap.delete(this.hashSubject(privateSubject)); - // If it didn't exist than we need to mint new token - if (!subjectExistedInPreviousState) { - tokensToBeMinted += 1; - } - this.insertTripleSorted( - publicGroup, - this.generatePrivateRepresentation(privateSubject), - ); - mergedTriples.push(publicGroup); - publicIndex++; - privateIndex++; - } - } - - // Append any remaining public triples - while (publicIndex < publicLength) { - const [publicSubject] = publicGroup[0].split(' '); - const subjectExistedInPreviousState = - subjectUALMap.delete(publicSubject) || - subjectHashUALMap.delete(this.hashSubject(publicSubject)); - // If it didn't exist than we need to mint new token - if (!subjectExistedInPreviousState) { - tokensToBeMinted += 1; - } - mergedTriples.push(...publicTriplesGrouped[publicIndex]); - publicIndex++; - } - - // Append any remaining private triples - while (privateIndex < privateLength) { - const [privateSubject] = privateTriplesGrouped[privateIndex][0].split(' '); - const subjectExistedInPreviousState = - subjectUALMap.delete(privateSubject) || - subjectHashUALMap.delete(this.hashSubject(privateSubject)); - // If it didn't exist than we need to mint new token - if (!subjectExistedInPreviousState) { - tokensToBeMinted += 1; - } - mergedTriples.push([this.generatePrivateRepresentation(privateSubject)]); - privateIndex++; - } - // Update the public dataset with the merged triples - dataset.public = mergedTriples.flat(); - } else { - // If there's no private dataset, ensure public is grouped correctly - dataset.public = kcTools.groupNquadsBySubject(dataset.public, true).flat(); - - for (const [publicTriple] of dataset.public) { - const [publicSubject] = publicTriple[0].split(' '); - const subjectExistedInPreviousState = - subjectUALMap.delete(publicSubject) || - subjectHashUALMap.delete(this.hashSubject(publicSubject)); - if (!subjectExistedInPreviousState) { - tokensToBeMinted += 1; - } - } } - // Find all old subject UALs that are not in new dataset - const tokensToBeBurned = new Set([...subjectUALMap.values(), ...subjectHashUALMap.values()]) - .map((ual) => ual.split('/').pop()) - .sort(); + // Finding new and old UAL creating private data representations const numberOfChunks = kcTools.calculateNumberOfChunks(dataset, CHUNK_BYTE_SIZE); From 31627889264a918d58b458c0ef4fc7043ced0678 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Fri, 13 Dec 2024 19:33:18 +0100 Subject: [PATCH 09/13] wip --- managers/asset-operations-manager.js | 97 +++++++++++++++++++++++++--- 1 file changed, 89 insertions(+), 8 deletions(-) diff --git a/managers/asset-operations-manager.js b/managers/asset-operations-manager.js index 5de882e..8bbe15d 100644 --- a/managers/asset-operations-manager.js +++ b/managers/asset-operations-manager.js @@ -13,7 +13,6 @@ import { CHUNK_BYTE_SIZE, OPERATION_DELAYS, PRIVATE_RESOURCE_PREDICATE, - PRIVATE_HASH_SUBJECT_PREFIX, PRIVATE_ASSERTION_PREDICATE, PRIVATE_RESOURCE_PREDICATE, PRIVATE_HASH_SUBJECT_PREFIX, @@ -1026,15 +1025,15 @@ export default class AssetOperationsManager { } }); + let privateTriplesGrouped = []; + // Handle inserting private root triple in new public // Private exist in new assertion if (dataset.private?.length) { dataset.private = kcTools.generateMissingIdsForBlankNodes(dataset.private); - const privateTriplesGrouped = kcTools.groupNquadsBySubject(dataset.private, true); + privateTriplesGrouped = kcTools.groupNquadsBySubject(dataset.private, true); dataset.private = privateTriplesGrouped.flat(); - const privateRoot = kcTools.calculateMerkleRoot( - privateTriplesGrouped.flat(dataset.private), - ); + const privateRoot = kcTools.calculateMerkleRoot(dataset.private); let publicTriplesGrouped = []; if (dataset.public?.length) { @@ -1044,7 +1043,6 @@ export default class AssetOperationsManager { if (privateMerkleRootTriple) { publicTriplesGrouped = kcTools.groupNquadsBySubject(dataset.public, true); const [privateMerkleRootTripleSubject] = privateMerkleRootTriple.split(' '); - let privateMerkleRootTripleExists = false; // This is sorted is should probably be done by binary search for (const [index, [triple]] of publicTriplesGrouped.entries()) { const [subject] = triple.split(' '); @@ -1055,7 +1053,6 @@ export default class AssetOperationsManager { publicTriplesGrouped[index][ changePrivateRootIndex ] = `${privateMerkleRootTripleSubject} <${PRIVATE_ASSERTION_PREDICATE}> "${privateRoot}" .`; - privateMerkleRootTripleExists = true; } } // Private root triple didn't existed before insert it @@ -1066,8 +1063,92 @@ export default class AssetOperationsManager { publicTriplesGrouped = kcTools.groupNquadsBySubject(dataset.public, true); } } - + let tokensCount = 0; + let tokensToBeMinted = 0; + let tokensToBeBurned = []; // Finding new and old UAL creating private data representations + // Private exists in new data + if (dataset.public?.length) { + if (dataset.private?.length) { + const publicSubjectMap = publicTriplesGrouped.reduce((map, group, index) => { + const [publicSubject] = group[0].split(' '); + map.set(publicSubject, index); + return map; + }, new Map()); + // Find if private subject has a public pair + const privateTriplesGroupedWithoutPublicPair = []; + const privateTripleSubjectHashesGroupedWithoutPublicPair = []; + for (const [privateIndex, privateTriples] of privateTriplesGrouped.entries()) { + const [privateSubject] = privateTriples[0].split(' '); + const privateSubjectHash = ethers.solidityPackedSha256( + ['string'], + [privateSubject.slice(1, -1)], + ); + + if (publicSubjectMap.has(privateSubject)) { + const publicIndex = publicSubjectMap.get(privateSubject); + this.insertTripleSorted( + publicTriplesGrouped[publicIndex], + this.generatePrivateRepresentation(privateSubjectHash), + ); + } else { + const index = this.insertTripleSorted( + privateTripleSubjectHashesGroupedWithoutPublicPair, + privateSubjectHash, + ); + privateTriplesGroupedWithoutPublicPair.splice( + index, + 0, + this.generatePrivateRepresentation(privateSubjectHash), + ); + } + } + // At the end of public append new triple arrays + + // Here I have all public + + tokensCount += privateTriplesGroupedWithoutPublicPair.length; + publicTriplesGrouped.push(...privateTriplesGroupedWithoutPublicPair); + dataset.public = publicTriplesGrouped.flat(); + } else { + publicTriplesGrouped = kcTools.groupNquadsBySubject(dataset.public, true); + tokensCount += publicTriplesGrouped.length; + dataset.public = publicTriplesGrouped.flat(); + // Find tokens to burn and mint + for (const [publicTriple] of publicTriplesGrouped) { + const [publicSubject] = publicTriple.split(' '); + // Check if this is recourse or hash + + // For previous state I have all resources that existed and hashes of public and private + if (subjectUALMap.has() || subjectHashUALMap.has()) { + } + } + } + } + // There is no public triples + else { + const privateTriplesGroupedWithoutPublicPair = []; + const privateTripleSubjectHashesGroupedWithoutPublicPair = []; + for (const { privateTriple, privateIndex } of privateTriplesGrouped.entries()) { + const [privateSubject] = privateTriple.split(' '); + const privateSubjectHash = ethers.solidityPackedSha256( + ['string'], + [privateSubject.slice(1, -1)], + ); + const index = this.insertTripleSorted( + privateTripleSubjectHashesGroupedWithoutPublicPair, + privateSubjectHash, + ); + privateTriplesGroupedWithoutPublicPair.splice( + index, + 0, + privateTriplesGrouped[privateIndex], + ); + } + // Count of private tokens + private root + tokensCount += privateTripleSubjectHashesGroupedWithoutPublicPair.length + 1; + dataset.public.push(...privateTripleSubjectHashesGroupedWithoutPublicPair); + } const numberOfChunks = kcTools.calculateNumberOfChunks(dataset, CHUNK_BYTE_SIZE); From aa1415016f47f809767593572c7445fe68bf9e9c Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Fri, 13 Dec 2024 23:57:08 +0100 Subject: [PATCH 10/13] maybe finished --- managers/asset-operations-manager.js | 81 ++++++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 3 deletions(-) diff --git a/managers/asset-operations-manager.js b/managers/asset-operations-manager.js index 8bbe15d..bd7b7b6 100644 --- a/managers/asset-operations-manager.js +++ b/managers/asset-operations-manager.js @@ -1003,6 +1003,7 @@ export default class AssetOperationsManager { null, ); + // Add error hadnling if this failes updated failes const subjectUALPairsResult = await this.nodeApiService.getOperationResult( endpoint, port, @@ -1109,6 +1110,39 @@ export default class AssetOperationsManager { tokensCount += privateTriplesGroupedWithoutPublicPair.length; publicTriplesGrouped.push(...privateTriplesGroupedWithoutPublicPair); + for (const [publicTriple] of publicTriplesGrouped) { + const [publicSubject] = publicTriple.split(' '); + // Check if this is recourse or hash + if (publicSubject.startsWith(`<${PRIVATE_HASH_SUBJECT_PREFIX}`)) { + const publicSubjectParsed = publicSubject.trim( + PRIVATE_HASH_SUBJECT_PREFIX.length + 1, + -1, + ); + if (!subjectHashUALMap.has(publicSubjectParsed)) { + tokensToBeMinted += 1; + } else { + subjectHashUALMap.delete(publicSubjectParsed); + } + } else { + const publicSubjectParsed = publicSubject.trim(1 - 1); + if (!subjectUALMap.has(publicSubjectParsed)) { + const publicSubjectHahsed = ethers.solidityPackedSha256( + ['string'], + [publicSubjectParsed], + ); + if (!subjectHashUALMap.has(publicSubjectHahsed)) { + tokensToBeMinted += 1; + } else { + subjectHashUALMap.delete(publicSubjectHahsed); + } + } else { + subjectUALMap.delete(publicSubjectParsed); + } + } + } + tokensToBeBurned = [ + ...Set(...subjectHashUALMap.values(), ...subjectUALMap().values), + ].map((ual) => ual.split('/').pop()); dataset.public = publicTriplesGrouped.flat(); } else { publicTriplesGrouped = kcTools.groupNquadsBySubject(dataset.public, true); @@ -1118,11 +1152,36 @@ export default class AssetOperationsManager { for (const [publicTriple] of publicTriplesGrouped) { const [publicSubject] = publicTriple.split(' '); // Check if this is recourse or hash - - // For previous state I have all resources that existed and hashes of public and private - if (subjectUALMap.has() || subjectHashUALMap.has()) { + if (publicSubject.startsWith(`<${PRIVATE_HASH_SUBJECT_PREFIX}`)) { + const publicSubjectParsed = publicSubject.trim( + PRIVATE_HASH_SUBJECT_PREFIX.length + 1, + -1, + ); + if (!subjectHashUALMap.has(publicSubjectParsed)) { + tokensToBeMinted += 1; + } else { + subjectHashUALMap.delete(publicSubjectParsed); + } + } else { + const publicSubjectParsed = publicSubject.trim(1 - 1); + if (!subjectUALMap.has(publicSubjectParsed)) { + const publicSubjectHahsed = ethers.solidityPackedSha256( + ['string'], + [publicSubjectParsed], + ); + if (!subjectHashUALMap.has(publicSubjectHahsed)) { + tokensToBeMinted += 1; + } else { + subjectHashUALMap.delete(publicSubjectHahsed); + } + } else { + subjectUALMap.delete(publicSubjectParsed); + } } } + tokensToBeBurned = [ + ...Set(...subjectHashUALMap.values(), ...subjectUALMap().values), + ].map((ual) => ual.split('/').pop()); } } // There is no public triples @@ -1148,6 +1207,22 @@ export default class AssetOperationsManager { // Count of private tokens + private root tokensCount += privateTripleSubjectHashesGroupedWithoutPublicPair.length + 1; dataset.public.push(...privateTripleSubjectHashesGroupedWithoutPublicPair); + for (const publicTriple of privateTripleSubjectHashesGroupedWithoutPublicPair) { + const [publicSubject] = publicTriple.split(' '); + // Check if this is recourse or hash + const publicSubjectParsed = publicSubject.trim( + PRIVATE_HASH_SUBJECT_PREFIX.length + 1, + -1, + ); + if (!subjectHashUALMap.has(publicSubjectParsed)) { + tokensToBeMinted += 1; + } else { + subjectHashUALMap.delete(publicSubjectParsed); + } + } + tokensToBeBurned = [ + ...Set(...subjectHashUALMap.values(), ...subjectUALMap().values), + ].map((ual) => ual.split('/').pop()); } const numberOfChunks = kcTools.calculateNumberOfChunks(dataset, CHUNK_BYTE_SIZE); From f9e02f286c55651cac5e7a40d16e7be124278c82 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Sat, 14 Dec 2024 12:49:52 +0100 Subject: [PATCH 11/13] Parsing should be done at this point --- managers/asset-operations-manager.js | 297 +++++++++++---------------- 1 file changed, 124 insertions(+), 173 deletions(-) diff --git a/managers/asset-operations-manager.js b/managers/asset-operations-manager.js index bd7b7b6..3830417 100644 --- a/managers/asset-operations-manager.js +++ b/managers/asset-operations-manager.js @@ -603,10 +603,6 @@ export default class AssetOperationsManager { )}>`} <${PRIVATE_RESOURCE_PREDICATE}> <${kaTools.generateNamedNode()}> .`; } - generatePrivateRepresentation(privateSubjectHash) { - return `${`<${PRIVATE_HASH_SUBJECT_PREFIX}${privateSubjectHash}>`} <${PRIVATE_RESOURCE_PREDICATE}> <${kaTools.generateNamedNode()}> .`; - } - /** * Transfer an asset to a new owner on a specified blockchain. * @async @@ -1003,7 +999,7 @@ export default class AssetOperationsManager { null, ); - // Add error hadnling if this failes updated failes + // TODO: Add error hadnling if this failes updated failes const subjectUALPairsResult = await this.nodeApiService.getOperationResult( endpoint, port, @@ -1013,216 +1009,171 @@ export default class AssetOperationsManager { frequency, getOperationId, ); - const subjectUALMap = new Map(); - const subjectHashUALMap = new Map(); + + // Initialize maps for quick lookups + const oldSubjectUALMap = new Map(); + const oldSubjectHashUALMap = new Map(); const { subjectUALPairs, privateMerkleRootTriple } = subjectUALPairsResult; - subjectUALPairs.forEach((pair) => { + + // Populate subjectUALMap and subjectHashUALMap from the pairs + for (const pair of subjectUALPairs) { if (pair.subject) { - subjectUALMap.set(pair.subject, pair.UAL); + oldSubjectUALMap.set(pair.subject, pair.UAL); } if (pair.privateSubjectHash) { - subjectHashUALMap.set(pair.privateSubjectHash, pair.UAL); + oldSubjectHashUALMap.set(pair.privateSubjectHash, pair.UAL); } - }); + } let privateTriplesGrouped = []; + let publicTriplesGrouped = []; - // Handle inserting private root triple in new public - // Private exist in new assertion + // Handle private root update/insertion if (dataset.private?.length) { dataset.private = kcTools.generateMissingIdsForBlankNodes(dataset.private); privateTriplesGrouped = kcTools.groupNquadsBySubject(dataset.private, true); dataset.private = privateTriplesGrouped.flat(); const privateRoot = kcTools.calculateMerkleRoot(dataset.private); - let publicTriplesGrouped = []; - if (dataset.public?.length) { - dataset.public = kcTools.generateMissingIdsForBlankNodes(dataset.public); - } - // Private root triple existed before update it if (privateMerkleRootTriple) { - publicTriplesGrouped = kcTools.groupNquadsBySubject(dataset.public, true); - const [privateMerkleRootTripleSubject] = privateMerkleRootTriple.split(' '); - // This is sorted is should probably be done by binary search - for (const [index, [triple]] of publicTriplesGrouped.entries()) { - const [subject] = triple.split(' '); - if (privateMerkleRootTripleSubject === subject) { - const changePrivateRootIndex = publicTriplesGrouped[index].findIndex( - PRIVATE_RESOURCE_PREDICATE, - ); - publicTriplesGrouped[index][ - changePrivateRootIndex - ] = `${privateMerkleRootTripleSubject} <${PRIVATE_ASSERTION_PREDICATE}> "${privateRoot}" .`; - } - } - // Private root triple didn't existed before insert it + // Private root triple existed before, so we update it + let newPrivateMerkleRootTriple = privateMerkleRootTriple.split(' '); + newPrivateMerkleRootTriple[2] = `"${privateRoot}"`; + dataset.public.push(newPrivateMerkleRootTriple.join(' ')); } else { + // Private root triple didn't exist, insert it dataset.public.push( `<${kaTools.generateNamedNode()}> <${PRIVATE_ASSERTION_PREDICATE}> "${privateRoot}" .`, ); - publicTriplesGrouped = kcTools.groupNquadsBySubject(dataset.public, true); } } + let tokensCount = 0; let tokensToBeMinted = 0; let tokensToBeBurned = []; - // Finding new and old UAL creating private data representations - // Private exists in new data - if (dataset.public?.length) { - if (dataset.private?.length) { - const publicSubjectMap = publicTriplesGrouped.reduce((map, group, index) => { - const [publicSubject] = group[0].split(' '); - map.set(publicSubject, index); - return map; - }, new Map()); - // Find if private subject has a public pair - const privateTriplesGroupedWithoutPublicPair = []; - const privateTripleSubjectHashesGroupedWithoutPublicPair = []; - for (const [privateIndex, privateTriples] of privateTriplesGrouped.entries()) { - const [privateSubject] = privateTriples[0].split(' '); - const privateSubjectHash = ethers.solidityPackedSha256( - ['string'], - [privateSubject.slice(1, -1)], - ); - if (publicSubjectMap.has(privateSubject)) { - const publicIndex = publicSubjectMap.get(privateSubject); - this.insertTripleSorted( - publicTriplesGrouped[publicIndex], - this.generatePrivateRepresentation(privateSubjectHash), - ); - } else { - const index = this.insertTripleSorted( - privateTripleSubjectHashesGroupedWithoutPublicPair, - privateSubjectHash, - ); - privateTriplesGroupedWithoutPublicPair.splice( - index, - 0, - this.generatePrivateRepresentation(privateSubjectHash), - ); - } + publicTriplesGrouped = kcTools.groupNquadsBySubject(dataset.public, true); + // We always have public as there is private root triple here already + if (dataset.private?.length) { + // Both public and private data exist + const publicSubjectMap = new Map(); + + for (let i = 0; i < publicTriplesGrouped.length; i++) { + const [publicSubject] = publicTriplesGrouped[i][0].split(' '); + publicSubjectMap.set(publicSubject, i); + } + + const privateTriplesGroupedWithoutPublicPair = []; + const privateTripleSubjectHashesGroupedWithoutPublicPair = []; + + // Integrate private subjects into public or create new entries if no public pair + for (const privateTriples of privateTriplesGrouped) { + const [privateSubject] = privateTriples[0].split(' '); + const privateSubjectHash = ethers.solidityPackedSha256( + ['string'], + [privateSubject.slice(1, -1)], + ); + + if (publicSubjectMap.has(privateSubject)) { + // Insert private representation into existing public subject group + const publicIndex = publicSubjectMap.get(privateSubject); + this.insertTripleSorted( + publicTriplesGrouped[publicIndex], + `${`<${PRIVATE_HASH_SUBJECT_PREFIX}${privateSubjectHash}>`} <${PRIVATE_RESOURCE_PREDICATE}> <${kaTools.generateNamedNode()}> .`, + ); + } else { + // No matching public pair - insert as a new hashed subject + const index = this.insertTripleSorted( + privateTripleSubjectHashesGroupedWithoutPublicPair, + privateSubjectHash, + ); + privateTriplesGroupedWithoutPublicPair.splice(index, 0, [ + `${`<${PRIVATE_HASH_SUBJECT_PREFIX}${privateSubjectHash}>`} <${PRIVATE_RESOURCE_PREDICATE}> <${kaTools.generateNamedNode()}> .`, + ]); } - // At the end of public append new triple arrays + } - // Here I have all public + tokensCount += privateTriplesGroupedWithoutPublicPair.length; + publicTriplesGrouped.push(...privateTriplesGroupedWithoutPublicPair); - tokensCount += privateTriplesGroupedWithoutPublicPair.length; - publicTriplesGrouped.push(...privateTriplesGroupedWithoutPublicPair); - for (const [publicTriple] of publicTriplesGrouped) { - const [publicSubject] = publicTriple.split(' '); - // Check if this is recourse or hash - if (publicSubject.startsWith(`<${PRIVATE_HASH_SUBJECT_PREFIX}`)) { - const publicSubjectParsed = publicSubject.trim( - PRIVATE_HASH_SUBJECT_PREFIX.length + 1, - -1, - ); - if (!subjectHashUALMap.has(publicSubjectParsed)) { - tokensToBeMinted += 1; - } else { - subjectHashUALMap.delete(publicSubjectParsed); - } + // Determine tokens to mint or burn + for (const triples of publicTriplesGrouped) { + // Each group is an array of triples, check only the first triple for subject + const [publicSubject] = triples[0].split(' '); + + // Public subject is hash + if (publicSubject.startsWith(`<${PRIVATE_HASH_SUBJECT_PREFIX}`)) { + // Subject is a hashed private subject + const publicSubjectParsed = publicSubject.slice( + PRIVATE_HASH_SUBJECT_PREFIX.length + 1, + -1, + ); + if (!oldSubjectHashUALMap.has(publicSubjectParsed)) { + // If didn't exist before it should be minted + tokensToBeMinted += 1; } else { - const publicSubjectParsed = publicSubject.trim(1 - 1); - if (!subjectUALMap.has(publicSubjectParsed)) { - const publicSubjectHahsed = ethers.solidityPackedSha256( - ['string'], - [publicSubjectParsed], - ); - if (!subjectHashUALMap.has(publicSubjectHahsed)) { - tokensToBeMinted += 1; - } else { - subjectHashUALMap.delete(publicSubjectHahsed); - } - } else { - subjectUALMap.delete(publicSubjectParsed); - } + // If existed remove from the map so only those that don't exist in new assertion stay + oldSubjectHashUALMap.delete(publicSubjectParsed); } - } - tokensToBeBurned = [ - ...Set(...subjectHashUALMap.values(), ...subjectUALMap().values), - ].map((ual) => ual.split('/').pop()); - dataset.public = publicTriplesGrouped.flat(); - } else { - publicTriplesGrouped = kcTools.groupNquadsBySubject(dataset.public, true); - tokensCount += publicTriplesGrouped.length; - dataset.public = publicTriplesGrouped.flat(); - // Find tokens to burn and mint - for (const [publicTriple] of publicTriplesGrouped) { - const [publicSubject] = publicTriple.split(' '); - // Check if this is recourse or hash - if (publicSubject.startsWith(`<${PRIVATE_HASH_SUBJECT_PREFIX}`)) { - const publicSubjectParsed = publicSubject.trim( - PRIVATE_HASH_SUBJECT_PREFIX.length + 1, - -1, + } else { + // Subject is a public resource + const publicSubjectParsed = publicSubject.slice(1, -1); + if (!oldSubjectUALMap.has(publicSubjectParsed)) { + // Didn't exist in previous public, check if it existed in private + const publicSubjectHashed = ethers.solidityPackedSha256( + ['string'], + [publicSubjectParsed], ); - if (!subjectHashUALMap.has(publicSubjectParsed)) { + if (!oldSubjectHashUALMap.has(publicSubjectHashed)) { + // It didn't exist in previous private it should be minted tokensToBeMinted += 1; } else { - subjectHashUALMap.delete(publicSubjectParsed); + // If existed remove from the map so only those that don't exist in new assertion stay + oldSubjectHashUALMap.delete(publicSubjectHashed); } } else { - const publicSubjectParsed = publicSubject.trim(1 - 1); - if (!subjectUALMap.has(publicSubjectParsed)) { - const publicSubjectHahsed = ethers.solidityPackedSha256( - ['string'], - [publicSubjectParsed], - ); - if (!subjectHashUALMap.has(publicSubjectHahsed)) { - tokensToBeMinted += 1; - } else { - subjectHashUALMap.delete(publicSubjectHahsed); - } - } else { - subjectUALMap.delete(publicSubjectParsed); - } + // If existed remove from the map so only those that don't exist in new assertion stay + oldSubjectUALMap.delete(publicSubjectParsed); } } - tokensToBeBurned = [ - ...Set(...subjectHashUALMap.values(), ...subjectUALMap().values), - ].map((ual) => ual.split('/').pop()); } - } - // There is no public triples - else { - const privateTriplesGroupedWithoutPublicPair = []; - const privateTripleSubjectHashesGroupedWithoutPublicPair = []; - for (const { privateTriple, privateIndex } of privateTriplesGrouped.entries()) { - const [privateSubject] = privateTriple.split(' '); - const privateSubjectHash = ethers.solidityPackedSha256( - ['string'], - [privateSubject.slice(1, -1)], - ); - const index = this.insertTripleSorted( - privateTripleSubjectHashesGroupedWithoutPublicPair, - privateSubjectHash, - ); - privateTriplesGroupedWithoutPublicPair.splice( - index, - 0, - privateTriplesGrouped[privateIndex], - ); - } - // Count of private tokens + private root - tokensCount += privateTripleSubjectHashesGroupedWithoutPublicPair.length + 1; - dataset.public.push(...privateTripleSubjectHashesGroupedWithoutPublicPair); - for (const publicTriple of privateTripleSubjectHashesGroupedWithoutPublicPair) { - const [publicSubject] = publicTriple.split(' '); - // Check if this is recourse or hash - const publicSubjectParsed = publicSubject.trim( - PRIVATE_HASH_SUBJECT_PREFIX.length + 1, - -1, - ); - if (!subjectHashUALMap.has(publicSubjectParsed)) { - tokensToBeMinted += 1; + + tokensToBeBurned = Array.from( + new Set([...oldSubjectHashUALMap.values(), ...oldSubjectUALMap.values()]), + ).map((ual) => ual.split('/').pop()); + + dataset.public = publicTriplesGrouped.flat(); + } else { + // No private triples, just handle public + tokensCount += publicTriplesGrouped.length; + dataset.public = publicTriplesGrouped.flat(); + + // Determine tokens to mint or burn for public-only scenario no hashes will be present + for (const triples of publicTriplesGrouped) { + const [publicSubject] = triples[0].split(' '); + const publicSubjectParsed = publicSubject.slice(1, -1); + if (!oldSubjectUALMap.has(publicSubjectParsed)) { + // Didn't exist in previous public, check if it existed in private + const publicSubjectHashed = ethers.solidityPackedSha256( + ['string'], + [publicSubjectParsed], + ); + if (!oldSubjectHashUALMap.has(publicSubjectHashed)) { + // It didn't exist in previous private it should be minted + tokensToBeMinted += 1; + } else { + // If existed before remove from the map so only those that don't exist in new assertion stay + oldSubjectHashUALMap.delete(publicSubjectHashed); + } } else { - subjectHashUALMap.delete(publicSubjectParsed); + oldSubjectUALMap.delete(publicSubjectParsed); } } - tokensToBeBurned = [ - ...Set(...subjectHashUALMap.values(), ...subjectUALMap().values), - ].map((ual) => ual.split('/').pop()); + + tokensToBeBurned = Array.from( + new Set([...oldSubjectHashUALMap.values(), ...oldSubjectUALMap.values()]), + ).map((ual) => ual.split('/').pop()); } const numberOfChunks = kcTools.calculateNumberOfChunks(dataset, CHUNK_BYTE_SIZE); From e93a463d31bb89931f9701c8f9cd9b8779c7b7a0 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Sat, 14 Dec 2024 12:55:35 +0100 Subject: [PATCH 12/13] some fixes --- managers/asset-operations-manager.js | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/managers/asset-operations-manager.js b/managers/asset-operations-manager.js index 3830417..577beea 100644 --- a/managers/asset-operations-manager.js +++ b/managers/asset-operations-manager.js @@ -1010,6 +1010,17 @@ export default class AssetOperationsManager { getOperationId, ); + if (subjectUALPairsResult.status !== OPERATION_STATUSES.COMPLETED) { + return { + operation: { + getSubjectUALPairs: getOperationStatusObject( + subjectUALPairsResult, + getOperationId, + ), + }, + }; + } + // Initialize maps for quick lookups const oldSubjectUALMap = new Map(); const oldSubjectHashUALMap = new Map(); @@ -1053,6 +1064,7 @@ export default class AssetOperationsManager { let tokensToBeMinted = 0; let tokensToBeBurned = []; + dataset.private = kcTools.generateMissingIdsForBlankNodes(dataset.private); publicTriplesGrouped = kcTools.groupNquadsBySubject(dataset.public, true); // We always have public as there is private root triple here already if (dataset.private?.length) { @@ -1213,7 +1225,7 @@ export default class AssetOperationsManager { return { datasetRoot, operation: { - publish: getOperationStatusObject(updateOperationResult, updateOperationId), + update: getOperationStatusObject(updateOperationResult, updateOperationId), }, }; } From 120f1e122cae14587d69288ea440fd2c6e8470ee Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Sat, 14 Dec 2024 13:10:47 +0100 Subject: [PATCH 13/13] remove some redundancy --- managers/asset-operations-manager.js | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/managers/asset-operations-manager.js b/managers/asset-operations-manager.js index 577beea..2ae23a7 100644 --- a/managers/asset-operations-manager.js +++ b/managers/asset-operations-manager.js @@ -1077,7 +1077,6 @@ export default class AssetOperationsManager { } const privateTriplesGroupedWithoutPublicPair = []; - const privateTripleSubjectHashesGroupedWithoutPublicPair = []; // Integrate private subjects into public or create new entries if no public pair for (const privateTriples of privateTriplesGrouped) { @@ -1097,17 +1096,16 @@ export default class AssetOperationsManager { } else { // No matching public pair - insert as a new hashed subject const index = this.insertTripleSorted( - privateTripleSubjectHashesGroupedWithoutPublicPair, - privateSubjectHash, - ); - privateTriplesGroupedWithoutPublicPair.splice(index, 0, [ + privateTriplesGroupedWithoutPublicPair, `${`<${PRIVATE_HASH_SUBJECT_PREFIX}${privateSubjectHash}>`} <${PRIVATE_RESOURCE_PREDICATE}> <${kaTools.generateNamedNode()}> .`, - ]); + ); } } tokensCount += privateTriplesGroupedWithoutPublicPair.length; - publicTriplesGrouped.push(...privateTriplesGroupedWithoutPublicPair); + for (const triple of privateTriplesGroupedWithoutPublicPair) { + publicTriplesGrouped.push([triple]); + } // Determine tokens to mint or burn for (const triples of publicTriplesGrouped) {