diff --git a/data-access/cert-api-client.js b/data-access/cert-api-client.js index 7c9c7c6..c553123 100644 --- a/data-access/cert-api-client.js +++ b/data-access/cert-api-client.js @@ -1,8 +1,10 @@ 'use strict'; const axios = require('axios'); +const chalk = require('chalk'); require('dotenv').config(); const { CERTIFICATION_API_KEY, ORGS_DATA_URL, SYSTEMS_DATA_URL } = process.env; +const AWS = require('aws-sdk'); const API_DEBOUNCE_SECONDS = 0.1; @@ -107,7 +109,7 @@ const processDataDictionaryResults = async ({ }; const getOrgsMap = async () => { - const { Data: orgs = [] } = (await axios.get(ORGS_DATA_URL)).data; + const { Organizations: orgs = [] } = (await axios.get(ORGS_DATA_URL)).data; if (!orgs?.length) throw new Error('ERROR: could not fetch Org data!'); @@ -157,10 +159,58 @@ const getOrgSystemsMap = async () => { }, {}); }; +const postToS3 = async ({ metadataReport, dataAvailabilityReport, s3Path, bucketName }) => { + if (!s3Path) { + throw new Error('Invalid S3 path'); + } + if (!bucketName) { + throw new Error('Invalid bucket name'); + } + + AWS.config.update({ + accessKeyId: process.env.ACCESS_KEY, + secretAccessKey: process.env.YOUR_SECRET_KEY, + region: process.env.AWS_REGION + }); + + const s3 = new AWS.S3(); + + const promises = []; + + if (metadataReport) { + const params = { + Bucket: bucketName, + Key: `${s3Path}/metadata-report.json`, + Body: metadataReport + }; + promises.push(s3.upload(params).promise()); + } + + if (dataAvailabilityReport) { + const params = { + Bucket: bucketName, + Key: `${s3Path}/data-availability-report.json`, + Body: dataAvailabilityReport + }; + promises.push(s3.upload(params).promise()); + } + + const response = await Promise.allSettled(promises); + + response.forEach(r => { + if (r.status === 'fulfilled') { + console.log(chalk.green(`File uploaded successfully. ${r.value.Location}`)); + } else { + console.error(chalk.redBright.bold(`Error uploading file to S3: ${r.reason}`)); + } + }); +}; + module.exports = { processDataDictionaryResults, getOrgsMap, getOrgSystemsMap, findDataDictionaryReport, - sleep + sleep, + postToS3 }; diff --git a/index.js b/index.js index c6b153f..b3e2734 100755 --- a/index.js +++ b/index.js @@ -12,6 +12,7 @@ program .command('restore') .option('-p, --pathToResults ', 'Path to test results') .option('-u, --url ', 'URL of Certification API') + .option('-b, --bucketName ', 'Name of the backup S3 bucket') .description('Restores local or S3 results to a RESO Certification API instance') .action(restore); diff --git a/utils/restore-utils/index.js b/utils/restore-utils/index.js index 0d31b20..cde0f7b 100644 --- a/utils/restore-utils/index.js +++ b/utils/restore-utils/index.js @@ -7,7 +7,8 @@ const { resolve, join } = require('path'); const { getOrgsMap, getOrgSystemsMap, - processDataDictionaryResults + processDataDictionaryResults, + postToS3 } = require('../../data-access/cert-api-client'); const { processLookupResourceMetadataFiles } = require('reso-certification-etl'); @@ -104,7 +105,7 @@ const restore = async (options = {}) => { missingResultsFilePaths: [] }; - const { pathToResults, url } = options; + const { pathToResults, url, bucketName = 'reso-backup' } = options; if (isS3Path(pathToResults)) { console.log( @@ -254,6 +255,18 @@ const restore = async (options = {}) => { metadataReport, dataAvailabilityReport }); + + await postToS3({ + metadataReport: Object.keys(metadataReport).length + ? JSON.stringify(metadataReport) + : null, + dataAvailabilityReport: Object.keys(dataAvailabilityReport).length + ? JSON.stringify(dataAvailabilityReport) + : null, + s3Path: `data_dictionary-${metadataReport.version}/${providerUoi}-${providerUsi}/${recipientUoi}`, + bucketName + }); + console.log(chalk.bold(`Done! Result: ${result ? 'Succeeded!' : 'Failed!'}`)); } catch (err) { console.log(chalk.bgRed.bold(err)); @@ -283,9 +296,7 @@ const restore = async (options = {}) => { console.log(chalk.bold(`Processing complete! Time Taken: ~${timeTaken}s`)); console.log(chalk.magentaBright.bold('------------------------------------------------------------')); - console.log( - chalk.bold(`\nItems Processed: ${STATS.processed.length} of ${totalItems}`) - ); + console.log(chalk.bold(`\nItems Processed: ${STATS.processed.length} of ${totalItems}`)); STATS.processed.forEach(item => console.log(chalk.bold(`\t * ${item}`))); console.log(chalk.bold(`\nProvider UOI Paths Skipped: ${STATS.skippedProviderUoiPaths.length}`));