Data loader for SSR [WIP]

CHANGES.txt

@@ -1,6 +1,9 @@
 2.0.0 (October XX, 2024)
  - Added support for targeting rules based on large segments.
  - Added `factory.destroy()` method, which invokes the `destroy` method on all SDK clients created by the factory.
+ - Added `factory.getState()` method for standalone server-side SDKs, which returns the rollout plan snapshot from the storage.
+ - Added `preloadedData` configuration option for standalone client-side SDKs, which allows preloading the SDK storage with a snapshot of the rollout plan.
+ - Updated internal storage factory to emit the SDK_READY_FROM_CACHE event when it corresponds, to clean up the initialization flow.
  - Updated the handling of timers and async operations inside an `init` factory method to enable lazy initialization of the SDK in standalone mode. This update is intended for the React SDK.
  - Bugfixing - Fixed an issue with the server-side polling manager that caused dangling timers when the SDK was destroyed before it was ready.
  - BREAKING CHANGES:

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@splitsoftware/splitio-commons",
-  "version": "2.0.0-rc.1",
+  "version": "2.0.0-rc.2",
   "description": "Split JavaScript SDK common components",
   "main": "cjs/index.js",
   "module": "esm/index.js",

src/sdkFactory/index.ts

@@ -7,7 +7,7 @@ import { IBasicClient, SplitIO } from '../types';
 import { validateAndTrackApiKey } from '../utils/inputValidation/apiKey';
 import { createLoggerAPI } from '../logger/sdkLogger';
 import { NEW_FACTORY, RETRIEVE_MANAGER } from '../logger/constants';
-import { SDK_SPLITS_ARRIVED, SDK_SEGMENTS_ARRIVED } from '../readiness/constants';
+import { SDK_SPLITS_ARRIVED, SDK_SEGMENTS_ARRIVED, SDK_SPLITS_CACHE_LOADED } from '../readiness/constants';
 import { objectAssign } from '../utils/lang/objectAssign';
 import { strategyDebugFactory } from '../trackers/strategy/strategyDebug';
 import { strategyOptimizedFactory } from '../trackers/strategy/strategyOptimized';
@@ -43,7 +43,7 @@ export function sdkFactory(params: ISdkFactoryParams): SplitIO.ICsSDK | SplitIO.
 
   const storage = storageFactory({
     settings,
-    onReadyCb: (error) => {
+    onReadyCb(error) {
       if (error) {
         // If storage fails to connect, SDK_READY_TIMED_OUT event is emitted immediately. Review when timeout and non-recoverable errors are reworked
         readiness.timeout();
@@ -52,8 +52,11 @@ export function sdkFactory(params: ISdkFactoryParams): SplitIO.ICsSDK | SplitIO.
       readiness.splits.emit(SDK_SPLITS_ARRIVED);
       readiness.segments.emit(SDK_SEGMENTS_ARRIVED);
     },
+    onReadyFromCacheCb() {
+      readiness.splits.emit(SDK_SPLITS_CACHE_LOADED);
+    }
   });
-  // @TODO add support for dataloader: `if (params.dataLoader) params.dataLoader(storage);`
+
   const clients: Record<string, IBasicClient> = {};
   const telemetryTracker = telemetryTrackerFactory(storage.telemetry, platform.now);
   const integrationsManager = integrationsManagerFactory && integrationsManagerFactory({ settings, storage, telemetryTracker });

src/storages/__tests__/dataLoader.spec.ts

@@ -0,0 +1,31 @@
+import { InMemoryStorageFactory } from '../inMemory/InMemoryStorage';
+import { InMemoryStorageCSFactory } from '../inMemory/InMemoryStorageCS';
+import { fullSettings } from '../../utils/settingsValidation/__tests__/settings.mocks';
+
+import * as dataLoader from '../dataLoader';
+
+test('loadData & getSnapshot', () => {
+  jest.spyOn(dataLoader, 'loadData');
+  const onReadyFromCacheCb = jest.fn();
+  // @ts-expect-error
+  const serverStorage = InMemoryStorageFactory({ settings: fullSettings });
+  serverStorage.splits.setChangeNumber(123); // @ts-expect-error
+  serverStorage.splits.addSplits([['split1', { name: 'split1' }]]);
+  serverStorage.segments.update('segment1', [fullSettings.core.key as string], [], 123);
+
+  const preloadedData = dataLoader.getSnapshot(serverStorage, [fullSettings.core.key as string]);
+
+  // @ts-expect-error
+  const clientStorage = InMemoryStorageCSFactory({ settings: { ...fullSettings, preloadedData }, onReadyFromCacheCb });
+
+  // Assert
+  expect(dataLoader.loadData).toBeCalledTimes(1);
+  expect(onReadyFromCacheCb).toBeCalledTimes(1);
+  expect(dataLoader.getSnapshot(clientStorage, [fullSettings.core.key as string])).toEqual(preloadedData);
+  expect(preloadedData).toEqual({
+    since: 123,
+    splitsData: [{ name: 'split1' }],
+    membershipsData: { [fullSettings.core.key as string]: { ms: { k: [{ n: 'segment1' }] }, ls: { k: [] } } },
+    segmentsData: undefined
+  });
+});

src/storages/dataLoader.ts

@@ -1,55 +1,114 @@
 import { SplitIO } from '../types';
-import { DEFAULT_CACHE_EXPIRATION_IN_MILLIS } from '../utils/constants/browser';
-import { DataLoader, ISegmentsCacheSync, ISplitsCacheSync } from './types';
+import { ISegmentsCacheSync, ISplitsCacheSync, IStorageSync } from './types';
+import { setToArray } from '../utils/lang/sets';
+import { getMatching } from '../utils/key';
+import { IMembershipsResponse, IMySegmentsResponse } from '../dtos/types';
 
 /**
- * Factory of client-side storage loader
+ * Storage-agnostic adaptation of `loadDataIntoLocalStorage` function
+ * (https://github.com/godaddy/split-javascript-data-loader/blob/master/src/load-data.js)
  *
- * @param preloadedData validated data following the format proposed in https://github.com/godaddy/split-javascript-data-loader
- * and extended with a `mySegmentsData` property.
- * @returns function to preload the storage
+ * @param preloadedData validated data following the format proposed in https://github.com/godaddy/split-javascript-data-loader and extended with a `mySegmentsData` property.
+ * @param storage object containing `splits` and `segments` cache (client-side variant)
+ * @param userKey user key (matching key) of the provided MySegmentsCache
+ *
+ * @TODO extend to load largeSegments
+ * @TODO extend to load data on shared mySegments storages. Be specific when emitting SDK_READY_FROM_CACHE on shared clients. Maybe the serializer should provide the `useSegments` flag.
+ * @TODO add logs, and input validation in this module, in favor of size reduction.
+ * @TODO unit tests
  */
-export function dataLoaderFactory(preloadedData: SplitIO.PreloadedData): DataLoader {
-
-  /**
-   * Storage-agnostic adaptation of `loadDataIntoLocalStorage` function
-   * (https://github.com/godaddy/split-javascript-data-loader/blob/master/src/load-data.js)
-   *
-   * @param storage object containing `splits` and `segments` cache (client-side variant)
-   * @param userId user key string of the provided MySegmentsCache
-   *
-   * @TODO extend to support SegmentsCache (server-side variant) by making `userId` optional and adding the corresponding logic.
-   * @TODO extend to load data on shared mySegments storages. Be specific when emitting SDK_READY_FROM_CACHE on shared clients. Maybe the serializer should provide the `useSegments` flag.
-   */
-  return function loadData(storage: { splits: ISplitsCacheSync, segments: ISegmentsCacheSync }, userId: string) {
-    // Do not load data if current preloadedData is empty
-    if (Object.keys(preloadedData).length === 0) return;
-
-    const { lastUpdated = -1, segmentsData = {}, since = -1, splitsData = {} } = preloadedData;
+export function loadData(preloadedData: SplitIO.PreloadedData, storage: { splits?: ISplitsCacheSync, segments: ISegmentsCacheSync, largeSegments?: ISegmentsCacheSync }, matchingKey?: string) {
+  // Do not load data if current preloadedData is empty
+  if (Object.keys(preloadedData).length === 0) return;
+
+  const { segmentsData = {}, since = -1, splitsData = [] } = preloadedData;
 
+  if (storage.splits) {
     const storedSince = storage.splits.getChangeNumber();
-    const expirationTimestamp = Date.now() - DEFAULT_CACHE_EXPIRATION_IN_MILLIS;
 
-    // Do not load data if current localStorage data is more recent,
-    // or if its `lastUpdated` timestamp is older than the given `expirationTimestamp`,
-    if (storedSince > since || lastUpdated < expirationTimestamp) return;
+    // Do not load data if current data is more recent
+    if (storedSince > since) return;
 
     // cleaning up the localStorage data, since some cached splits might need be part of the preloaded data
     storage.splits.clear();
     storage.splits.setChangeNumber(since);
 
     // splitsData in an object where the property is the split name and the pertaining value is a stringified json of its data
-    storage.splits.addSplits(Object.keys(splitsData).map(splitName => JSON.parse(splitsData[splitName])));
-
-    // add mySegments data
-    let mySegmentsData = preloadedData.mySegmentsData && preloadedData.mySegmentsData[userId];
-    if (!mySegmentsData) {
-      // segmentsData in an object where the property is the segment name and the pertaining value is a stringified object that contains the `added` array of userIds
-      mySegmentsData = Object.keys(segmentsData).filter(segmentName => {
-        const userIds = JSON.parse(segmentsData[segmentName]).added;
-        return Array.isArray(userIds) && userIds.indexOf(userId) > -1;
-      });
+    storage.splits.addSplits(splitsData.map(split => ([split.name, split])));
+  }
+
+  if (matchingKey) { // add mySegments data (client-side)
+    let membershipsData = preloadedData.membershipsData && preloadedData.membershipsData[matchingKey];
+    if (!membershipsData && segmentsData) {
+      membershipsData = {
+        ms: {
+          k: Object.keys(segmentsData).filter(segmentName => {
+            const segmentKeys = segmentsData[segmentName];
+            return segmentKeys.indexOf(matchingKey) > -1;
+          }).map(segmentName => ({ n: segmentName }))
+        }
+      };
+    }
+    if (membershipsData) {
+      if (membershipsData.ms) storage.segments.resetSegments(membershipsData.ms);
+      if (membershipsData.ls && storage.largeSegments) storage.largeSegments.resetSegments(membershipsData.ls);
     }
-    storage.segments.resetSegments({ k: mySegmentsData.map(s => ({ n: s })) });
+
+  } else { // add segments data (server-side)
+    Object.keys(segmentsData).forEach(segmentName => {
+      const segmentKeys = segmentsData[segmentName];
+      storage.segments.update(segmentName, segmentKeys, [], -1);
+    });
+  }
+}
+
+export function getSnapshot(storage: IStorageSync, userKeys?: SplitIO.SplitKey[]): SplitIO.PreloadedData {
+  return {
+    // lastUpdated: Date.now(),
+    since: storage.splits.getChangeNumber(),
+    splitsData: storage.splits.getAll(),
+    segmentsData: userKeys ?
+      undefined : // @ts-ignore accessing private prop
+      Object.keys(storage.segments.segmentCache).reduce((prev, cur) => { // @ts-ignore accessing private prop
+        prev[cur] = setToArray(storage.segments.segmentCache[cur] as Set<string>);
+        return prev;
+      }, {}),
+    membershipsData: userKeys ?
+      userKeys.reduce<Record<string, IMembershipsResponse>>((prev, userKey) => {
+        if (storage.shared) {
+          // Client-side segments
+          // @ts-ignore accessing private prop
+          const sharedStorage = storage.shared(userKey);
+          prev[getMatching(userKey)] = {
+            ms: {
+              // @ts-ignore accessing private prop
+              k: Object.keys(sharedStorage.segments.segmentCache).map(segmentName => ({ n: segmentName })),
+              // cn: sharedStorage.segments.getChangeNumber()
+            },
+            ls: sharedStorage.largeSegments ? {
+              // @ts-ignore accessing private prop
+              k: Object.keys(sharedStorage.largeSegments.segmentCache).map(segmentName => ({ n: segmentName })),
+              // cn: sharedStorage.largeSegments.getChangeNumber()
+            } : undefined
+          };
+        } else {
+          prev[getMatching(userKey)] = {
+            ms: {
+              // Server-side segments
+              // @ts-ignore accessing private prop
+              k: Object.keys(storage.segments.segmentCache).reduce<IMySegmentsResponse['k']>((prev, segmentName) => { // @ts-ignore accessing private prop
+                return storage.segments.segmentCache[segmentName].has(userKey) ?
+                  prev!.concat({ n: segmentName }) :
+                  prev;
+              }, [])
+            },
+            ls: {
+              k: []
+            }
+          };
+        }
+        return prev;
+      }, {}) :
+      undefined
   };
 }

src/storages/inMemory/InMemoryStorageCS.ts

@@ -7,14 +7,16 @@ import { ImpressionCountsCacheInMemory } from './ImpressionCountsCacheInMemory';
 import { DEBUG, LOCALHOST_MODE, NONE, STORAGE_MEMORY } from '../../utils/constants';
 import { shouldRecordTelemetry, TelemetryCacheInMemory } from './TelemetryCacheInMemory';
 import { UniqueKeysCacheInMemoryCS } from './UniqueKeysCacheInMemoryCS';
+import { getMatching } from '../../utils/key';
+import { loadData } from '../dataLoader';
 
 /**
  * InMemory storage factory for standalone client-side SplitFactory
  *
  * @param params parameters required by EventsCacheSync
  */
 export function InMemoryStorageCSFactory(params: IStorageFactoryParams): IStorageSync {
-  const { settings: { scheduler: { impressionsQueueSize, eventsQueueSize, }, sync: { impressionsMode, __splitFiltersValidation } } } = params;
+  const { settings: { scheduler: { impressionsQueueSize, eventsQueueSize, }, sync: { impressionsMode, __splitFiltersValidation }, preloadedData }, onReadyFromCacheCb } = params;
 
   const splits = new SplitsCacheInMemory(__splitFiltersValidation);
   const segments = new MySegmentsCacheInMemory();
@@ -42,11 +44,18 @@ export function InMemoryStorageCSFactory(params: IStorageFactoryParams): IStorag
     },
 
     // When using shared instantiation with MEMORY we reuse everything but segments (they are unique per key)
-    shared() {
+    shared(matchingKey: string) {
+      const segments = new MySegmentsCacheInMemory();
+      const largeSegments = new MySegmentsCacheInMemory();
+
+      if (preloadedData) {
+        loadData(preloadedData, { segments, largeSegments }, matchingKey);
+      }
+
       return {
         splits: this.splits,
-        segments: new MySegmentsCacheInMemory(),
-        largeSegments: new MySegmentsCacheInMemory(),
+        segments,
+        largeSegments,
         impressions: this.impressions,
         impressionCounts: this.impressionCounts,
         events: this.events,
@@ -72,6 +81,12 @@ export function InMemoryStorageCSFactory(params: IStorageFactoryParams): IStorag
     if (storage.uniqueKeys) storage.uniqueKeys.track = noopTrack;
   }
 
+
+  if (preloadedData) {
+    loadData(preloadedData, storage, getMatching(params.settings.core.key));
+    if (splits.getChangeNumber() > -1) onReadyFromCacheCb();
+  }
+
   return storage;
 }
 

src/storages/types.ts

@@ -488,15 +488,17 @@ export interface IStorageAsync extends IStorageBase<
 
 /** StorageFactory */
 
-export type DataLoader = (storage: IStorageSync, matchingKey: string) => void
-
 export interface IStorageFactoryParams {
   settings: ISettings,
   /**
    * Error-first callback invoked when the storage is ready to be used. An error means that the storage failed to connect and shouldn't be used.
    * It is meant for emitting SDK_READY event in consumer mode, and waiting before using the storage in the synchronizer.
    */
   onReadyCb: (error?: any) => void,
+  /**
+   * It is meant for emitting SDK_READY_FROM_CACHE event in standalone mode with preloaded data
+   */
+  onReadyFromCacheCb: () => void,
 }
 
 export type StorageType = 'MEMORY' | 'LOCALSTORAGE' | 'REDIS' | 'PLUGGABLE';

src/types.ts

@@ -1,4 +1,4 @@
-import { ISplitFiltersValidation } from './dtos/types';
+import { IMembershipsResponse, ISplit, ISplitFiltersValidation } from './dtos/types';
 import { IIntegration, IIntegrationFactoryParams } from './integrations/types';
 import { ILogger } from './logger/types';
 import { ISdkFactoryContext } from './sdkFactory/types';
@@ -97,6 +97,7 @@ export interface ISettings {
     eventsFirstPushWindow: number
   },
   readonly storage: IStorageSyncFactory | IStorageAsyncFactory,
+  readonly preloadedData?: SplitIO.PreloadedData,
   readonly integrations: Array<{
     readonly type: string,
     (params: IIntegrationFactoryParams): IIntegration | void
@@ -770,31 +771,31 @@ export namespace SplitIO {
      * If this value is older than 10 days ago (expiration time policy), the data is not used to update the storage content.
      * @TODO configurable expiration time policy?
      */
-    lastUpdated: number,
+    // lastUpdated: number,
     /**
      * Change number of the preloaded data.
      * If this value is older than the current changeNumber at the storage, the data is not used to update the storage content.
      */
     since: number,
     /**
-     * Map of feature flags to their stringified definitions.
+     * List of feature flag definitions.
+     * @TODO rename to flags
      */
-    splitsData: {
-      [splitName: string]: string
-    },
+    splitsData: ISplit[],
     /**
-     * Optional map of user keys to their list of segments.
-     * @TODO remove when releasing first version
+     * Optional map of user keys to their memberships.
+     * @TODO rename to memberships
      */
-    mySegmentsData?: {
-      [key: string]: string[]
+    membershipsData?: {
+      [key: string]: IMembershipsResponse
     },
     /**
-     * Optional map of segments to their stringified definitions.
-     * This property is ignored if `mySegmentsData` was provided.
+     * Optional map of segments to their list of keys.
+     * This property is ignored if `membershipsData` was provided.
+     * @TODO rename to segments
      */
     segmentsData?: {
-      [segmentName: string]: string
+      [segmentName: string]: string[]
     },
   }
   /**