#35: create webapisync option in cert utils

common.js

@@ -44,14 +44,10 @@ const availableVersions = {
 };
 
 const getCurrentVersion = endorsementName =>
-  endorsementName &&
-  availableVersions[endorsementName] &&
-  availableVersions[endorsementName].currentVersion;
+  endorsementName && availableVersions[endorsementName] && availableVersions[endorsementName].currentVersion;
 
 const getPreviousVersion = endorsementName =>
-  endorsementName &&
-  availableVersions[endorsementName] &&
-  availableVersions[endorsementName].previousVersion;
+  endorsementName && availableVersions[endorsementName] && availableVersions[endorsementName].previousVersion;
 
 /**
  * Determines whether the given endorsementName is valid.
@@ -60,8 +56,7 @@ const getPreviousVersion = endorsementName =>
  * @returns true if the endorsementName is valid, false otherwise.
  * @throws error if parameters aren't valid
  */
-const isValidEndorsement = endorsementName =>
-  endorsementName && !!availableVersions[endorsementName];
+const isValidEndorsement = endorsementName => endorsementName && !!availableVersions[endorsementName];
 
 /**
  * Determines whether the version is valid for the given endorsement.
@@ -90,9 +85,7 @@ const getEndorsementMetadata = (endorsementName, version) => {
   }
 
   if (!isValidVersion(endorsementName, version)) {
-    throw new Error(
-      `Invalid endorsement version! endorsementKey: ${endorsementName}, version: ${version}`
-    );
+    throw new Error(`Invalid endorsement version! endorsementKey: ${endorsementName}, version: ${version}`);
   }
 
   const ddVersion = version || CURRENT_DATA_DICTIONARY_VERSION,
@@ -104,10 +97,7 @@ const getEndorsementMetadata = (endorsementName, version) => {
       version: `${ddVersion}`,
       /* TODO: add versions to JSON results file names in the Commander */
       jsonResultsFiles: [METADATA_REPORT_JSON, DATA_AVAILABILITY_REPORT_JSON],
-      htmlReportFiles: [
-        `data-dictionary-${ddVersion}.html`,
-        `data-availability.dd-${ddVersion}.html`
-      ],
+      htmlReportFiles: [`data-dictionary-${ddVersion}.html`, `data-availability.dd-${ddVersion}.html`],
       logFileName: COMMANDER_LOG_FILE_NAME
     };
   }
@@ -117,11 +107,7 @@ const getEndorsementMetadata = (endorsementName, version) => {
       directoryName: `${endorsements.DATA_DICTIONARY_WITH_IDX}`,
       version: `${version}`,
       /* TODO: add versions to JSON results file names in the Commander */
-      jsonResultsFiles: [
-        METADATA_REPORT_JSON,
-        DATA_AVAILABILITY_REPORT_JSON,
-        IDX_DIFFERENCE_REPORT_JSON
-      ],
+      jsonResultsFiles: [METADATA_REPORT_JSON, DATA_AVAILABILITY_REPORT_JSON, IDX_DIFFERENCE_REPORT_JSON],
       htmlReportFiles: [
         `data-dictionary-${ddVersion}.html`,
         `data-availability.dd-${ddVersion}.html`,
@@ -191,15 +177,10 @@ const buildRecipientEndorsementPath = ({
   if (!endorsementName) throw Error('endorsementName is required!');
   if (!version) throw Error('version is required!');
 
-  if (!isValidEndorsement(endorsementName))
-    throw new Error(`Invalid endorsementName: ${endorsementName}`);
+  if (!isValidEndorsement(endorsementName)) throw new Error(`Invalid endorsementName: ${endorsementName}`);
   if (!isValidVersion(endorsementName, version)) throw new Error(`Invalid version: ${version}`);
 
-  return path.join(
-    `${providerUoi}-${providerUsi}`,
-    recipientUoi,
-    currentOrArchived
-  );
+  return path.join(`${providerUoi}-${providerUsi}`, recipientUoi, currentOrArchived);
 };
 
 /**
@@ -212,13 +193,7 @@ const buildRecipientEndorsementPath = ({
  * @param {String} providerUsi
  * @param {String} recipientUoi
  */
-const archiveEndorsement = ({
-  providerUoi,
-  providerUsi,
-  recipientUoi,
-  endorsementName,
-  version
-} = {}) => {
+const archiveEndorsement = ({ providerUoi, providerUsi, recipientUoi, endorsementName, version } = {}) => {
   const currentRecipientPath = buildRecipientEndorsementPath({
     providerUoi,
     providerUsi,
@@ -270,13 +245,20 @@ const createResoScriptClientCredentialsConfig = ({ serviceRootUri, clientCredent
   `    <ClientSecret>${clientCredentials.clientSecret}</ClientSecret>` +
   `    <TokenURI>${clientCredentials.tokenUri}</TokenURI>` +
   `    ${
-    clientCredentials.scope
-      ? '<ClientScope>' + clientCredentials.scope + '</ClientScope>'
-      : EMPTY_STRING
+    clientCredentials.scope ? '<ClientScope>' + clientCredentials.scope + '</ClientScope>' : EMPTY_STRING
   }` +
   '  </ClientSettings>' +
   '</OutputScript>';
 
+const checkFileExists = async filePath => {
+  try {
+    await fs.promises.access(filePath);
+    return true;
+  } catch (err) {
+    return false;
+  }
+};
+
 module.exports = {
   endorsements,
   availableVersions,
@@ -291,5 +273,6 @@ module.exports = {
   getCurrentVersion,
   getPreviousVersion,
   CURRENT_DATA_DICTIONARY_VERSION,
-  CURRENT_WEB_API_CORE_VERSION
+  CURRENT_WEB_API_CORE_VERSION,
+  checkFileExists
 };

data-access/cert-api-client.js

@@ -46,6 +46,66 @@ const postDataDictionaryResultsToApi = async ({
   }
 };
 
+const postWebAPIResultsToApi = async ({
+  url,
+  providerUoi,
+  providerUsi,
+  recipientUoi,
+  webAPIReport = {}
+} = {}) => {
+  if (!url) throw new Error('url is required!');
+  if (!providerUoi) throw new Error('providerUoi is required!');
+  if (!providerUsi) throw new Error('providerUsi is required!');
+  if (!recipientUoi) throw new Error('recipientUoi is required!');
+  if (!Object.keys(webAPIReport)?.length) throw new Error('web API report is empty!');
+
+  try {
+    const { id: reportId = null } =
+      (
+        await axios.post(
+          `${url}/api/v1/certification_reports/web_api_server_core/${providerUoi}`,
+          webAPIReport,
+          {
+            maxContentLength: Infinity,
+            maxBodyLength: Infinity,
+            headers: {
+              Authorization: `ApiKey ${CERTIFICATION_API_KEY}`,
+              recipientUoi,
+              'Content-Type': 'application/json',
+              providerUsi
+            }
+          }
+        )
+      ).data || {};
+
+    if (!reportId) throw new Error('Did not receive the required id parameter from the response!');
+
+    return reportId;
+  } catch (err) {
+    throw new Error(`Could not post web API results to API! ${err}`);
+  }
+};
+
+const deleteDataDictionaryResults = async ({ url, reportId } = {}) => {
+  if (!url) throw new Error('url is required!');
+  if (!reportId) throw new Error('reportId is required!');
+
+  try {
+    const { deletedReport } =
+      (
+        await axios.delete(`${url}/api/v1/certification_reports/${reportId}`, {
+          headers: {
+            Authorization: `ApiKey ${CERTIFICATION_API_KEY}`
+          }
+        })
+      ).data || {};
+
+    return deletedReport;
+  } catch (err) {
+    throw new Error(err);
+  }
+};
+
 const postDataAvailabilityResultsToApi = async ({ url, reportId, dataAvailabilityReport = {} } = {}) => {
   if (!url) throw new Error('url is required!');
   if (!reportId) throw new Error('reportId is required!');
@@ -106,8 +166,40 @@ const processDataDictionaryResults = async ({
   }
 };
 
+const processWebAPIResults = async ({
+  url,
+  providerUoi,
+  providerUsi,
+  recipientUoi,
+  webAPIReport = {},
+  reportIdToDelete,
+  overwrite = false
+}) => {
+  try {
+    if (overwrite) {
+      if (!reportIdToDelete) throw new Error('reportIdToDelete MUST be present when overwrite is used!');
+      await deleteDataDictionaryResults({ url, reportId: reportIdToDelete });
+    }
+
+    //wait for the dust to settle to avoid thrashing the server
+    await sleep(API_DEBOUNCE_SECONDS * 1000);
+
+    const reportId = await postWebAPIResultsToApi({
+      url,
+      providerUoi,
+      providerUsi,
+      recipientUoi,
+      webAPIReport
+    });
+
+    return reportId;
+  } catch (err) {
+    throw new Error(`Could not process webAPI results! ${err}`);
+  }
+};
+
 const getOrgsMap = async () => {
-  const { Data: orgs = [] } = (await axios.get(ORGS_DATA_URL)).data;
+  const { Organizations: orgs = [] } = (await axios.get(ORGS_DATA_URL)).data;
 
   if (!orgs?.length) throw new Error('ERROR: could not fetch Org data!');
 
@@ -149,6 +241,29 @@ const findDataDictionaryReport = async ({ serverUrl, providerUoi, providerUsi, r
   }
 };
 
+const findWebAPIReport = async ({ serverUrl, providerUoi, providerUsi, recipientUoi } = {}) => {
+  const url = `${serverUrl}/api/v1/certification_reports/summary/${recipientUoi}`,
+    config = {
+      headers: {
+        Authorization: `ApiKey ${CERTIFICATION_API_KEY}`
+      }
+    };
+
+  try {
+    const { data = [] } = await axios.get(url, config);
+
+    return data.find(
+      item =>
+        item?.type === 'web_api_server_core' &&
+        item?.providerUoi === providerUoi &&
+        //provider USI isn't in the data set at the moment, only filter if it's present
+        (item?.providerUsi ? item.providerUsi === providerUsi : true)
+    );
+  } catch (err) {
+    throw new Error(`Could not connect to ${url}`);
+  }
+};
+
 const getOrgSystemsMap = async () => {
   return (await axios.get(SYSTEMS_DATA_URL))?.data?.values.slice(1).reduce((acc, [providerUoi, , usi]) => {
     if (!acc[providerUoi]) acc[providerUoi] = [];
@@ -157,10 +272,21 @@ const getOrgSystemsMap = async () => {
   }, {});
 };
 
+const getSystemsMap = async () => {
+  return (await axios.get(SYSTEMS_DATA_URL))?.data?.values.slice(1).reduce((acc, [providerUoi, , usi]) => {
+    acc[usi] = providerUoi;
+    return acc;
+  }, {});
+};
+
 module.exports = {
   processDataDictionaryResults,
   getOrgsMap,
   getOrgSystemsMap,
   findDataDictionaryReport,
-  sleep
+  sleep,
+  findWebAPIReport,
+  postWebAPIResultsToApi,
+  processWebAPIResults,
+  getSystemsMap
 };

index.js

@@ -1,7 +1,8 @@
 #! /usr/bin/env node
 const { program } = require('commander');
-const { restore } = require('./utils/restore-utils');
+const { restore } = require('./utils/restore-utils/data-dictionary');
 const { runTests } = require('./utils/batch-test-runner');
+const { syncWebApi } = require('./utils/restore-utils/web-api-core');
 
 program
   .name('reso-certification-utils')
@@ -15,10 +16,23 @@ program
   .description('Restores local or S3 results to a RESO Certification API instance')
   .action(restore);
 
+program
+  .command('syncWebApiResults')
+  .option('-p, --pathToResults <string>', 'Path to test results')
+  .option('-u, --url <string>', 'URL of Certification API')
+  .option('-o, --overwrite', 'Flag to overwrite existing passed files')
+  .option('-r, --recipients <string>', 'Comma-separated list of recipient orgs')
+  .option('-i, --system <string>', 'Unique system identifier')
+  .description('Restores local or S3 Web API results to a RESO Certification API instance')
+  .action(syncWebApi);
+
 program
   .command('runDDTests')
   .requiredOption('-p, --pathToConfigFile <string>', 'Path to config file')
-  .option('-a, --runAvailability', 'Flag to run data availability tests, otherwise only metadata tests are run')
+  .option(
+    '-a, --runAvailability',
+    'Flag to run data availability tests, otherwise only metadata tests are run'
+  )
   .description('Runs Data Dictionary tests')
   .action(runTests);
 

utils/restore-utils/index.js → utils/restore-utils/data-dictionary.js

@@ -11,6 +11,7 @@ const {
 } = require('../../data-access/cert-api-client');
 
 const { processLookupResourceMetadataFiles } = require('reso-certification-etl');
+const { checkFileExists } = require('../../common');
 
 const CERTIFICATION_RESULTS_DIRECTORY = 'current',
   PATH_DATA_SEPARATOR = '-';
@@ -89,7 +90,10 @@ const isValidUrl = url => {
 
 /**
  * Restores a RESO Certification Server from either a local or S3 path.
- * @param {String} path
+ * @param {Object} options
+ * @param {string} options.pathToResults An absolute local path to the DD results or a valid S3 path.
+ * @param {string} options.url Cert API base URL.
+ * @param {boolean} options.overwrite Overwrite option - when true the program will overwrite the existing reports on the Cert API.
  * @throws Error if path is not a valid S3 or local path
  */
 const restore = async (options = {}) => {
@@ -221,11 +225,13 @@ const restore = async (options = {}) => {
                       pathToOutputFile = resolve(
                         join(currentResultsPath, CERTIFICATION_FILES.PROCESSED_METADATA_REPORT)
                       );
-                    await processLookupResourceMetadataFiles(
-                      pathToMetadataReport,
-                      pathToLookupResourceData,
-                      pathToOutputFile
-                    );
+                    if (!(await checkFileExists(pathToOutputFile))) {
+                      await processLookupResourceMetadataFiles(
+                        pathToMetadataReport,
+                        pathToLookupResourceData,
+                        pathToOutputFile
+                      );
+                    }
                   }
 
                   const metadataReport =
@@ -270,7 +276,6 @@ const restore = async (options = {}) => {
         }
       }
     }
-    console.log();
   } else {
     console.log(chalk.bgRedBright.bold(`Invalid path: ${pathToResults}! \nMust be valid S3 or local path`));
   }
@@ -283,9 +288,7 @@ const restore = async (options = {}) => {
   console.log(chalk.bold(`Processing complete! Time Taken: ~${timeTaken}s`));
   console.log(chalk.magentaBright.bold('------------------------------------------------------------'));
 
-  console.log(
-    chalk.bold(`\nItems Processed: ${STATS.processed.length} of ${totalItems}`)
-  );
+  console.log(chalk.bold(`\nItems Processed: ${STATS.processed.length} of ${totalItems}`));
   STATS.processed.forEach(item => console.log(chalk.bold(`\t * ${item}`)));
 
   console.log(chalk.bold(`\nProvider UOI Paths Skipped: ${STATS.skippedProviderUoiPaths.length}`));