Merge pull request #975 from Chia-Network/feat/951

feat: add mirror check task

src/datalayer/writeService.js

@@ -2,15 +2,13 @@ import _ from 'lodash';
 
 import * as dataLayer from './persistance';
 import wallet from './wallet';
-import fullNode from './fullNode';
 import * as simulator from './simulator';
-import { encodeHex } from '../utils/datalayer-utils';
+import { encodeHex, getMirrorUrl } from '../utils/datalayer-utils';
 import { getConfig } from '../utils/config-loader';
 import { logger } from '../config/logger.cjs';
 import { Organization } from '../models';
-import { publicIpv4 } from '../utils/ip-tools';
 
-const { USE_SIMULATOR, DATALAYER_FILE_SERVER_URL } = getConfig().APP;
+const { USE_SIMULATOR, AUTO_MIRROR_EXTERNAL_STORES } = getConfig().APP;
 
 const createDataLayerStore = async () => {
   await wallet.waitForAllTransactionsToConfirm();
@@ -27,14 +25,14 @@ const createDataLayerStore = async () => {
     await waitForStoreToBeConfirmed(storeId);
     await wallet.waitForAllTransactionsToConfirm();
 
-    const chiaConfig = fullNode.getChiaConfig();
+    // Default AUTO_MIRROR_EXTERNAL_STORES to true if it is null or undefined
+    // This make sure this runs by default even if the config param is missing
+    const shouldMirror = AUTO_MIRROR_EXTERNAL_STORES ?? true;
 
-    await dataLayer.addMirror(
-      storeId,
-      DATALAYER_FILE_SERVER_URL ||
-        `http://${await publicIpv4()}:${chiaConfig.data_layer.host_port}`,
-      true,
-    );
+    if (shouldMirror) {
+      const mirrorUrl = await getMirrorUrl();
+      await dataLayer.addMirror(storeId, mirrorUrl, true);
+    }
   }
 
   return storeId;

src/models/organizations/organizations.model.js

@@ -222,8 +222,8 @@ class Organization extends Model {
     return registryVersionId;
   }
 
-  static async addMirror(storeId, coinId) {
-    await datalayer.addMirror(storeId, coinId);
+  static async addMirror(storeId, url) {
+    await datalayer.addMirror(storeId, url);
   }
 
   static async importHomeOrg(orgUid) {

src/tasks/index.js

@@ -5,6 +5,7 @@ import syncPickLists from './sync-picklists';
 import syncRegistries from './sync-registries';
 import syncOrganizationMeta from './sync-organization-meta';
 import syncGovernanceBody from './sync-governance-body';
+import mirrorCheck from './mirror-check';
 
 const scheduler = new ToadScheduler();
 
@@ -23,6 +24,7 @@ const start = () => {
     syncPickLists,
     syncRegistries,
     syncOrganizationMeta,
+    mirrorCheck,
   ];
   defaultJobs.forEach((defaultJob) => {
     jobRegistry[defaultJob.id] = defaultJob;

src/tasks/mirror-check.js

@@ -0,0 +1,62 @@
+import { SimpleIntervalJob, Task } from 'toad-scheduler';
+import { Organization } from '../models';
+import {
+  assertDataLayerAvailable,
+  assertWalletIsSynced,
+} from '../utils/data-assertions';
+import { logger } from '../config/logger.cjs';
+import { getConfig } from '../utils/config-loader';
+import { getMirrorUrl } from '../utils/datalayer-utils';
+import dotenv from 'dotenv';
+
+const CONFIG = getConfig().APP;
+dotenv.config();
+
+// This task checks if there are any mirrors that have not been properly mirrored and then mirrors them if not
+
+const task = new Task('mirror-check', async () => {
+  try {
+    await assertDataLayerAvailable();
+    await assertWalletIsSynced();
+
+    // Default AUTO_MIRROR_EXTERNAL_STORES to true if it is null or undefined
+    const shouldMirror = CONFIG?.AUTO_MIRROR_EXTERNAL_STORES ?? true;
+
+    if (!CONFIG.USE_SIMULATOR && shouldMirror) {
+      runMirrorCheck();
+    }
+  } catch (error) {
+    logger.error(
+      `Retrying in ${CONFIG?.TASKS?.MIRROR_CHECK_TASK_INTERVAL || 300} seconds`,
+      error,
+    );
+  }
+});
+
+const job = new SimpleIntervalJob(
+  {
+    seconds: CONFIG?.TASKS?.MIRROR_CHECK_TASK_INTERVAL || 300,
+    runImmediately: true,
+  },
+  task,
+  { id: 'mirror-check', preventOverrun: true },
+);
+
+const runMirrorCheck = async () => {
+  const homeOrg = Organization.getHomeOrg();
+
+  if (homeOrg) {
+    const organizations = Organization.getOrgsMap();
+    const orgs = Object.keys(organizations);
+    for (const org of orgs) {
+      const orgData = organizations[org];
+      const mirrorUrl = await getMirrorUrl();
+
+      // There is logic within the addMirror function to check if the mirror already exists
+      await Organization.addMirror(orgData.orgUid, mirrorUrl);
+      await Organization.addMirror(orgData.registryId, mirrorUrl);
+    }
+  }
+};
+
+export default job;

src/utils/datalayer-utils.js

@@ -1,3 +1,7 @@
+import { getConfig } from './config-loader';
+import fullNode from '../datalayer/fullNode';
+import { publicIpv4 } from './ip-tools';
+
 export const encodeHex = (str) => {
   return Buffer.from(str).toString('hex');
 };
@@ -71,3 +75,12 @@ export const deserializeMaker = (maker) => {
 
   return changes;
 };
+
+export const getMirrorUrl = async () => {
+  const { DATALAYER_FILE_SERVER_URL } = getConfig().APP;
+  const chiaConfig = fullNode.getChiaConfig();
+  return (
+    DATALAYER_FILE_SERVER_URL ||
+    `http://${await publicIpv4()}:${chiaConfig.data_layer.host_port}`
+  );
+};

src/utils/defaultConfig.js

@@ -21,10 +21,12 @@ export const defaultConfig = {
     CERTIFICATE_FOLDER_PATH: null,
     DATALAYER_FILE_SERVER_URL: null,
     AUTO_SUBSCRIBE_FILESTORE: false,
+    AUTO_MIRROR_EXTERNAL_STORES: true,
     TASKS: {
       GOVERNANCE_SYNC_TASK_INTERVAL: 86400,
       ORGANIZATION_META_SYNC_TASK_INTERVAL: 300,
       PICKLIST_SYNC_TASK_INTERVAL: 30,
+      MIRROR_CHECK_TASK_INTERVAL: 300,
     },
   },
   GOVERNANCE: {