fix: governance fails to sync on initial start

src/controllers/organization.controller.js

@@ -37,7 +37,7 @@ export const homeOrgSyncStatus = async (req, res) => {
         home_org_synced: Boolean(homeOrg?.synced),
         pending_commits: pendingCommitsCount,
         home_org_profile_synced:
-          sync_status.target_root_hash === homeOrg.orgHash,
+          sync_status.target_root_hash === homeOrg.orgHash?.split('0x')?.[1],
       },
       success: true,
     });

src/datalayer/persistance.js

@@ -362,6 +362,12 @@ const getStoreData = async (storeId, rootHash) => {
             `datalayer get_keys_values returned no data for store ${storeId} at root hash: ${rootHash || 'latest'}`,
           );
         }
+
+        logger.trace(
+          `raw keys and values from RPC for store ${storeId}
+          
+          ${JSON.stringify(data.keys_values)}`,
+        );
         return data;
       } else {
         throw new Error(JSON.stringify(data));

src/datalayer/syncService.js

@@ -1,6 +1,9 @@
 import _ from 'lodash';
 
-import { decodeDataLayerResponse } from '../utils/datalayer-utils';
+import {
+  decodeDataLayerResponse,
+  isDlStoreSynced,
+} from '../utils/datalayer-utils';
 import { Simulator } from '../models';
 import { getConfig } from '../utils/config-loader';
 import { logger } from '../config/logger.js';
@@ -58,11 +61,13 @@ const subscribeToStoreOnDataLayer = async (storeId) => {
  *
  * @param storeId {string} to retrieve data from
  * @param providedSubscriptions {[string] | undefined} optional list of subscriptions. providing prevents RPC call
- * @returns {Promise<any>}
+ * @param waitForSync {boolean} option to block returning data until the store is synced. could be time expensive.
+ * @returns {Promise<object>}
  */
 const getSubscribedStoreData = async (
   storeId,
   providedSubscriptions = undefined,
+  waitForSync = false,
 ) => {
   let subscriptions = providedSubscriptions;
   if (!subscriptions) {
@@ -86,6 +91,21 @@ const getSubscribedStoreData = async (
     }
   }
 
+  if (waitForSync) {
+    let synced = false;
+    while (!synced) {
+      const syncStatus = dataLayer.getSyncStatus(storeId);
+      synced = isDlStoreSynced(syncStatus);
+
+      if (!synced) {
+        logger.warn(
+          `datalayer has not fully synced subscribed store ${storeId}. waiting to return data until store is synced`,
+        );
+        await new Promise((resolve) => setTimeout(() => resolve, 10000));
+      }
+    }
+  }
+
   logger.debug(`Subscription Found for ${storeId}.`);
 
   if (!USE_SIMULATOR) {
@@ -118,6 +138,12 @@ const getSubscribedStoreData = async (
   }
 
   const decodedData = decodeDataLayerResponse(encodedData);
+  logger.trace(
+    `the data for subscribed store ${storeId} after conversion to js Object is:
+    
+    ${JSON.stringify(decodedData)}`,
+  );
+
   if (!decodedData) {
     return {};
   }

src/models/governance/governance.model.js

@@ -69,7 +69,16 @@ class Governance extends Model {
     return governanceVersionId;
   }
 
-  static async upsertGovernanceDownload(governanceData) {
+  static async upsertGovernanceDownload(
+    sourceGovernanceBodyId,
+    governanceData,
+  ) {
+    if (!governanceData) {
+      throw new Error(
+        'upsertGovernanceDownload() received a nil or falsy governance data value',
+      );
+    }
+
     const updates = [];
 
     if (governanceData.orgList) {
@@ -78,6 +87,10 @@ class Governance extends Model {
         metaValue: governanceData.orgList,
         confirmed: true,
       });
+    } else {
+      logger.warn(
+        `governance data in store ${sourceGovernanceBodyId} does not contain orgList values`,
+      );
     }
 
     if (governanceData.glossary) {
@@ -86,6 +99,10 @@ class Governance extends Model {
         metaValue: governanceData.glossary,
         confirmed: true,
       });
+    } else {
+      logger.warn(
+        `governance data in store ${sourceGovernanceBodyId} does not contain glossary values`,
+      );
     }
 
     if (governanceData.pickList) {
@@ -103,13 +120,20 @@ class Governance extends Model {
         metaValue: JSON.stringify(PickListStub),
         confirmed: true,
       });
+    } else {
+      logger.warn(
+        `governance data in store ${sourceGovernanceBodyId} does not contain picklist values`,
+      );
     }
 
+    logger.debug('upserting governance data from governance body store');
     await Promise.all(updates.map(async (update) => Governance.upsert(update)));
   }
 
-  static async sync() {
+  static async sync(retryCounter = 0) {
     try {
+      logger.debug('running governance model sync()');
+
       if (!GOVERNANCE_BODY_ID) {
         throw new Error('Missing information in env to sync Governance data');
       }
@@ -126,33 +150,62 @@ class Governance extends Model {
         return;
       }
 
-      const governanceData =
-        await datalayer.getSubscribedStoreData(GOVERNANCE_BODY_ID);
+      const governanceData = await datalayer.getSubscribedStoreData(
+        GOVERNANCE_BODY_ID,
+        undefined,
+        true,
+      );
 
       // Check if there is v1, v2, v3 ..... and if not, then we assume this is a legacy governance table that isnt versioned
       const shouldSyncLegacy = !Object.keys(governanceData).some((key) =>
         /^v?[0-9]+$/.test(key),
       );
 
       if (shouldSyncLegacy) {
-        await Governance.upsertGovernanceDownload(governanceData);
+        logger.info(
+          `using legacy governance upsert method for governance store ${GOVERNANCE_BODY_ID}`,
+        );
+        await Governance.upsertGovernanceDownload(
+          GOVERNANCE_BODY_ID,
+          governanceData,
+        );
       }
 
       // Check if the governance data for this version exists
       const dataModelVersion = getDataModelVersion();
-      if (governanceData[dataModelVersion]) {
+      const versionedGovernanceStoreId = governanceData[dataModelVersion];
+      if (versionedGovernanceStoreId) {
+        logger.debug(
+          `getting ${dataModelVersion} governance data from store ${versionedGovernanceStoreId}`,
+        );
         const versionedGovernanceData = await datalayer.getSubscribedStoreData(
-          governanceData[dataModelVersion],
+          versionedGovernanceStoreId,
+          undefined,
+          true,
         );
 
-        await Governance.upsertGovernanceDownload(versionedGovernanceData);
+        await Governance.upsertGovernanceDownload(
+          GOVERNANCE_BODY_ID,
+          versionedGovernanceData,
+        );
       } else {
         throw new Error(
-          `Governance data is not available from this source for ${dataModelVersion} data model.`,
+          `Governance data is not available from store ${GOVERNANCE_BODY_ID} for ${dataModelVersion} data model.`,
         );
       }
     } catch (error) {
-      logger.error('Error Syncing Governance Data', error);
+      await new Promise((resolve) => setTimeout(resolve, 5000));
+      const maxRetry = 50;
+      if (retryCounter < maxRetry) {
+        logger.error(
+          `Error Syncing Governance Data. Retry attempt #${retryCounter + 1}. Retrying. Error:, ${error}`,
+        );
+        await Governance.sync(retryCounter + 1);
+      } else {
+        logger.error(
+          `Error Syncing Governance Data. Retry attempts exceeded. This will not have the latest governance data and data sync may be impacted`,
+        );
+      }
     }
   }
 

src/models/organizations/organizations.model.js

@@ -8,14 +8,14 @@ const { Model } = Sequelize;
 import { sequelize } from '../../database';
 
 import datalayer from '../../datalayer';
-import { logger } from '../../config/logger.js';
+import { logger } from '../../config/logger';
 import { Audit, FileStore, Meta, ModelKeys, Staging } from '../';
 import { getDataModelVersion } from '../../utils/helpers';
 import { getConfig } from '../../utils/config-loader';
 const { USE_SIMULATOR, AUTO_SUBSCRIBE_FILESTORE } = getConfig().APP;
 
 import ModelTypes from './organizations.modeltypes.cjs';
-import { assertStoreIsOwned } from '../../utils/data-assertions.js';
+import { assertStoreIsOwned } from '../../utils/data-assertions';
 import {
   getRoot,
   getSubscriptions,
@@ -24,7 +24,8 @@ import {
 import {
   addOrDeleteOrganizationRecordMutex,
   processingSyncRegistriesTransactionMutex,
-} from '../../utils/model-utils.js';
+  isDlStoreSynced,
+} from '../../utils/model-utils';
 
 class Organization extends Model {
   static async getHomeOrg(includeAddress = true) {
@@ -365,16 +366,12 @@ class Organization extends Model {
         datalayerDataModelVersionStoreId;
     }
 
-    const isDlSynced = (syncStatus) => {
-      return syncStatus.generation === syncStatus.target_generation;
-    };
-
     // note that we only update the data model version store hash here because the other two store hashes are updated elsewhere
     const dataModelVersionStoreSyncStatus = await getSyncStatus(
       datalayerDataModelVersionStoreId,
     );
 
-    if (isDlSynced(dataModelVersionStoreSyncStatus)) {
+    if (isDlStoreSynced(dataModelVersionStoreSyncStatus)) {
       const { confirmed, hash } = await getRoot(
         datalayerDataModelVersionStoreId,
       );
@@ -583,7 +580,11 @@ class Organization extends Model {
     let orgStoreData = null;
     while (!orgStoreData) {
       try {
-        orgStoreData = await datalayer.getSubscribedStoreData(orgUid);
+        orgStoreData = await datalayer.getSubscribedStoreData(
+          orgUid,
+          undefined,
+          true,
+        );
       } catch (error) {
         if (reachedTimeout()) {
           onTimeout(error);
@@ -610,6 +611,8 @@ class Organization extends Model {
       try {
         dataModelVersionStoreData = await datalayer.getSubscribedStoreData(
           dataModelVersionStoreId,
+          undefined,
+          true,
         );
       } catch (error) {
         if (reachedTimeout()) {

src/utils/data-loaders.js

@@ -80,3 +80,11 @@ export const serverAvailable = async (server, port) => {
     }
   }
 };
+
+export const isDlStoreSynced = (syncStatus) => {
+  if (syncStatus?.generation && syncStatus?.target_generation) {
+    return syncStatus.generation === syncStatus.target_generation;
+  }
+
+  return false;
+};