Merge remote-tracking branch 'origin/master' into feature/broker-ufe-…

…tests

packages/openactive-broker-microservice/package.json

@@ -19,10 +19,10 @@
     "@openactive/data-model-validator": "^2.0.78",
     "@openactive/data-models": "^2.0.316",
     "@openactive/dataset-utils": "^1.0.1",
-    "@openactive/harvesting-utils": "github:openactive/harvesting-utils#9dad411",
+    "@openactive/harvesting-utils": "github:openactive/harvesting-utils#1b2877834055549572fa059a491ac17d306942fd",
     "@openactive/openactive-openid-browser-automation": "file:../openactive-openid-browser-automation",
     "@openactive/openactive-openid-client": "file:../openactive-openid-client",
-    "@openactive/rpde-validator": "^2.0.19",
+    "@openactive/rpde-validator": "^2.0.20",
     "@openactive/test-interface-criteria": "file:../test-interface-criteria",
     "await-semaphore": "^0.1.3",
     "axios": "^0.19.2",

packages/openactive-broker-microservice/src/core.js

@@ -11,7 +11,7 @@ const cliProgress = require('cli-progress');
 const { validate } = require('@openactive/data-model-validator');
 const { expect } = require('chai');
 const { isNil, partialRight } = require('lodash');
-const { harvestRPDE, createFeedContext, progressFromContext } = require('@openactive/harvesting-utils');
+const { createFeedContext, progressFromContext, harvestRPDELossless } = require('@openactive/harvesting-utils');
 const { partial } = require('lodash');
 const path = require('path');
 
@@ -826,7 +826,7 @@ async function touchChildOpportunityItems(parentIds) {
   await Promise.all([...opportunitiesToUpdate].map(async (jsonLdId) => {
     if (state.opportunityItemRowCache.store.has(jsonLdId)) {
       const row = state.opportunityItemRowCache.store.get(jsonLdId);
-      row.feedModified = Date.now() + 1000; // 1 second in the future
+      row.feedModified = `${Date.now() + 1000}`; // 1 second in the future
       row.waitingForParentToBeIngested = false;
       await processRow(row);
     }
@@ -865,7 +865,7 @@ async function storeChildOpportunityItem(item) {
     id: item.id,
     modified: item.modified,
     deleted: false,
-    feedModified: Date.now() + 1000, // 1 second in the future,
+    feedModified: `${Date.now() + 1000}`, // 1 second in the future,
     jsonLdId: item.data['@id'] || item.data.id || null,
     jsonLd: item.data,
     jsonLdType: item.data['@type'] || item.data.type,
@@ -905,7 +905,7 @@ async function processRow(row) {
     newItem = {
       state: row.deleted ? 'deleted' : 'updated',
       id: row.jsonLdId,
-      modified: row.feedModified,
+      modified: `${row.feedModified}`,
       data: row.jsonLd,
     };
   } else {
@@ -925,7 +925,7 @@ async function processRow(row) {
     newItem = {
       state: row.deleted ? 'deleted' : 'updated',
       id: row.jsonLdId,
-      modified: row.feedModified,
+      modified: `${row.feedModified}`,
       data: {
         '@context': mergedContexts,
         ...rowJsonLdWithoutContext,
@@ -1256,7 +1256,7 @@ async function startPollingForOpportunityFeed(datasetDistributionItem, { validat
     state.incompleteFeeds.markFeedHarvestStarted(feedContextIdentifier);
     const ingestParentOpportunityPageForThisFeed = partialRight(ingestParentOpportunityPage, sendItemsToValidatorWorkerPoolForThisFeed);
 
-    await harvestRPDE({
+    await harvestRPDELossless({
       baseUrl: datasetDistributionItem.contentUrl,
       feedContextIdentifier,
       headers: withOpportunityRpdeHeaders(async () => OPPORTUNITY_FEED_REQUEST_HEADERS),
@@ -1290,7 +1290,7 @@ async function startPollingForOpportunityFeed(datasetDistributionItem, { validat
     state.incompleteFeeds.markFeedHarvestStarted(feedContextIdentifier);
     const ingestOpportunityPageForThisFeed = partialRight(ingestChildOpportunityPage, sendItemsToValidatorWorkerPoolForThisFeed);
 
-    await harvestRPDE({
+    await harvestRPDELossless({
       baseUrl: datasetDistributionItem.contentUrl,
       feedContextIdentifier,
       headers: withOpportunityRpdeHeaders(async () => OPPORTUNITY_FEED_REQUEST_HEADERS),
@@ -1339,7 +1339,7 @@ async function startPollingForOrderFeed(feedUrl, type, feedContextIdentifier, fe
   };
 
   state.incompleteFeeds.markFeedHarvestStarted(feedContextIdentifier);
-  await harvestRPDE({
+  await harvestRPDELossless({
     baseUrl: feedUrl,
     feedContextIdentifier,
     headers: withOrdersRpdeHeaders(getOrdersFeedHeader(feedBookingPartnerIdentifier)),

packages/openactive-broker-microservice/src/models/core.d.ts

@@ -16,7 +16,12 @@ export type RpdeItem = {
   state: string;
   kind: string;
   id: string | number;
-  modified: string | number;
+  /** modified handled as a string as it could be a number bigger than JS's number limit.
+   * This causes issues with fidelity and ordering. It is not stored as a BigInt in memory as lots of libraries have
+   * problems with BigInts. It is stored as a string and converted to a BigInt when comparisons are needed, and then
+   * converted back.
+   */
+  modified: string;
   data?: Opportunity;
 };
 
@@ -28,8 +33,8 @@ export type OpportunityItemRow = {
   id: RpdeItem['id'];
   modified: RpdeItem['modified'];
   deleted: boolean;
-  /** Timestamp (Date.now()) when item was ingested */
-  feedModified: number;
+  /** Timestamp (Date.now()) when item was ingested. Handled as a string as all modifieds are strings for consistency. For more information see above. */
+  feedModified: string;
   jsonLdId: string;
   jsonLd: Opportunity;
   /** e.g 'Slot' */

packages/openactive-broker-microservice/src/util/item-transforms.js

@@ -3,7 +3,7 @@ const { omit, isNil, isEmpty } = require('lodash');
 /**
  * Inverts any FacilityUse items that have an `individualFacilityUse` property, so that the top-level `kind` is "IndividualFacilityUse"
  * @typedef {{'@id'?: string, id?: string}} IndividualFacilityUse
- * @param {{state: string, modified:any, kind: string, id: string, data: {id: string, individualFacilityUse?: IndividualFacilityUse[] }}} item
+ * @param {{state: string, modified:string, kind: string, id: string, data: {id: string, individualFacilityUse?: IndividualFacilityUse[] }}} item
  */
 function invertFacilityUseItem(item) {
   if (!isNil(item.data?.individualFacilityUse) && !isEmpty(item.data.individualFacilityUse)) {
@@ -26,7 +26,7 @@ function invertFacilityUseItem(item) {
  * and using this function splits the subEvent into items to simulate a separate ScheduledSessions feed.
  *
  * @param {Record<string, any>} subEvent
- * @param {{modified: any, data: Record<string, any>}} item
+ * @param {{modified: string, data: Record<string, any>}} item
  */
 function createItemFromSubEvent(subEvent, item) {
   const opportunityItemData = {

packages/openactive-broker-microservice/test/user-facing-endpoints-test.js

@@ -22,9 +22,9 @@ const testDataGenerators = {
         jsonLd: {
           '@type': 'ScheduledSession',
         },
-        feedModified: Date.now(),
+        feedModified: String(Date.now()),
         deleted: false,
-        modified: 123,
+        modified: '123',
         waitingForParentToBeIngested: false,
       },
     ],
@@ -41,9 +41,9 @@ const testDataGenerators = {
         jsonLd: {
           '@type': 'ScheduledSession',
         },
-        feedModified: Date.now(),
+        feedModified: String(Date.now()),
         deleted: false,
-        modified: 123,
+        modified: '123',
         waitingForParentToBeIngested: false,
       },
     ],
@@ -61,9 +61,9 @@ const testDataGenerators = {
         jsonLd: {
           '@type': 'ScheduledSession',
         },
-        feedModified: Date.now(),
+        feedModified: String(Date.now()),
         deleted: false,
-        modified: 123,
+        modified: '123',
         waitingForParentToBeIngested: true,
       },
     ],
@@ -176,7 +176,7 @@ describe('user-facing endpoints', () => {
             jsonLd: {
               '@type': 'ScheduledSession',
             },
-            modified: 123,
+            modified: '123',
           }],
         },
       });