From b736fb80a528f78abc2c597bbc55da5dff9d65b8 Mon Sep 17 00:00:00 2001 From: DinoChiesa Date: Thu, 19 Dec 2024 06:30:52 +0000 Subject: [PATCH 1/2] feat: implement the download feature --- README.md | 41 +++++- cli.js | 257 +++++++++++++++++++++----------------- lib/package/downloader.js | 110 ++++++++++++++++ 3 files changed, 290 insertions(+), 118 deletions(-) create mode 100644 lib/package/downloader.js diff --git a/README.md b/README.md index bb4d0d1..f587e24 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ Usage: apigeelint [options] Options: -V, --version output the version number -s, --path Path of the proxy to analyze + -d, --download [value] Download the API proxy or sharedflow to analyze. Exclusive of -s / --path. Example: org:ORG,API:proxyname or org:ORG,sf:SHAREDFLOWNAME -f, --formatter [value] Specify formatters (default: json.js) -w, --write [value] file path to write results -e, --excluded [value] The comma separated list of tests to exclude (default: none) @@ -106,10 +107,48 @@ archive. This tool also can read and analyze these zipped bundles: apigeelint -f table.js -s path/to/your/apiproxy.zip ``` -The tool will unzip the bundle to a temporary directory, perform the analysis, +The tool will unzip the bundle into a temporary directory, perform the analysis, and then remove the temporary directory. +### Basic usage: downloading a proxy bundle to analyze + +You can ask apigeelint to export an API Proxy or Sharedflow bundle from Apigee, +and analyze the resulting zip archive. This connects to apigee.googleapis.com to +perform the export, which means it will work only with Apigee X or hybrid. + +``` +# to download and then analyze a proxy bundle +apigeelint -f table.js -d org:your-org-name,api:name-of-your-api-proxy + +# to download and then analyze a sharedflow bundle +apigeelint -f table.js -d org:your-org-name,sf:name-of-your-shared-flow +``` + +With this invocation, the tool will: +- obtain a token using the `gcloud auth print-access-token` command +- use the token to inquire the latest revision of the proxy or sharedflow +- use the token to download the bundle for the latest revision +- unzip the bundle into a temporary directory +- perform the lint analysis +- render the result +- and then remove the temporary directory + +If you do not have the [`gcloud` command line +tool](https://cloud.google.com/sdk/gcloud) installed, and available on your +path, this will fail. + + +You can also specify a token you have obtained previously: + +``` +apigeelint -f table.js -d org:your-org-name,api:name-of-your-api-proxy,token:ACCESS_TOKEN_HERE +``` + +In this case, apigeelint does not try to use `gcloud` to obtain an access token. + + + ### Using External Plugins: ``` apigeelint -x ./externalPlugins -s path/to/your/apiproxy -f table.js diff --git a/cli.js b/cli.js index a75a1a7..72795e4 100755 --- a/cli.js +++ b/cli.js @@ -24,6 +24,7 @@ const program = require("commander"), bl = require("./lib/package/bundleLinter.js"), rc = require("./lib/package/apigeelintrc.js"), pkj = require("./package.json"), + downloader = require("./lib/package/downloader.js"), bundleType = require("./lib/package/BundleTypes.js"), debug = require("debug"); @@ -77,136 +78,158 @@ const findBundle = (p) => { ); }; -program - .version(pkj.version) - .option( - "-s, --path ", - "Path of the exploded apiproxy or sharedflowbundle directory", - ) - .option("-f, --formatter [value]", "Specify formatters (default: json.js)") - .option("-w, --write [value]", "file path to write results") - .option( - "-e, --excluded [value]", - "The comma separated list of tests to exclude (default: none)", - ) - // .option("-M, --mgmtserver [value]", "Apigee management server") - // .option("-u, --user [value]", "Apigee user account") - // .option("-p, --password [value]", "Apigee password") - // .option("-o, --organization [value]", "Apigee organization") - .option( - "-x, --externalPluginsDirectory [value]", - "Relative or full path to an external plugins directory", - ) - .option( - "-q, --quiet", - "do not emit the report to stdout. (can use --write option to write to file)", - ) - .option( - "--list", - "do not execute, instead list the available plugins and formatters", - ) - .option( - "--maxWarnings [value]", - "Number of warnings to trigger nonzero exit code (default: -1)", - ) - .option("--profile [value]", "Either apigee or apigeex (default: apigee)") - .option( - "--norc", - "do not search for and use the .apigeelintrc file for settings", - ) - .option( - "--ignoreDirectives", - "ignore any directives within XML files that disable warnings", - ); +(async function main() { + program + .version(pkj.version) + .option( + "-s, --path ", + "Path of the exploded apiproxy or sharedflowbundle directory", + ) + .option( + "-d, --download [value]", + "Download the API proxy or sharedflow to analyze. Exclusive of -s / --path. Example: org:ORG,API:proxyname or org:ORG,sf:SHAREDFLOWNAME", + ) + .option("-f, --formatter [value]", "Specify formatters (default: json.js)") + .option("-w, --write [value]", "file path to write results") + .option( + "-e, --excluded [value]", + "The comma separated list of tests to exclude (default: none)", + ) + // .option("-M, --mgmtserver [value]", "Apigee management server") + // .option("-u, --user [value]", "Apigee user account") + // .option("-p, --password [value]", "Apigee password") + // .option("-o, --organization [value]", "Apigee organization") + .option( + "-x, --externalPluginsDirectory [value]", + "Relative or full path to an external plugins directory", + ) + .option( + "-q, --quiet", + "do not emit the report to stdout. (can use --write option to write to file)", + ) + .option( + "--list", + "do not scan, instead list the available plugins and formatters", + ) + .option( + "--maxWarnings [value]", + "Number of warnings to trigger nonzero exit code (default: -1)", + ) + .option("--profile [value]", "Either apigee or apigeex (default: apigee)") + .option( + "--norc", + "do not search for and use the .apigeelintrc file for settings", + ) + .option( + "--ignoreDirectives", + "ignore any directives within XML files that disable warnings", + ); -program.on("--help", function () { - console.log("\nExample: apigeelint -f table.js -s sampleProxy/apiproxy"); - console.log(""); -}); + program.on("--help", function () { + console.log("\nExamples:"); + console.log(" apigeelint -f table.js -s sampleProxy/apiproxy\n"); + console.log(" apigeelint -f table.js -s path/to/your/apiproxy.zip\n"); + console.log( + " apigeelint -f table.js --download org:my-org,api:my-proxy\n", + ); + console.log(""); + }); + + program.parse(process.argv); + + if (program.list) { + console.log("available plugins: " + bl.listRuleIds().join(", ") + "\n"); + console.log("available formatters: " + bl.listFormatters().join(", ")); + if (fs.existsSync(program.externalPluginsDirectory)) { + console.log( + "\n" + + "available external plugins: " + + bl.listExternalRuleIds(program.externalPluginsDirectory).join(", ") + + "\n", + ); + } + process.exit(0); + } -program.parse(process.argv); + if (program.download) { + if (program.path) { + console.log( + "you must specify either the -s /--path option, or the -d /--download option. Not both.", + ); + process.exit(1); + } + // This will work only with Apigee X/hybrid + program.path = await downloader.downloadBundle(program.download); + } -if (program.list) { - console.log("available plugins: " + bl.listRuleIds().join(", ") + "\n"); - console.log("available formatters: " + bl.listFormatters().join(", ")); - if (fs.existsSync(program.externalPluginsDirectory)) { + if (!program.path) { console.log( - "\n" + - "available external plugins: " + - bl.listExternalRuleIds(program.externalPluginsDirectory).join(", ") + - "\n", + "you must specify the -s option, or the long form of that: --path ", ); + process.exit(1); } - process.exit(0); -} -if (!program.path) { - console.log( - "you must specify the -s option, or the long form of that: --path ", - ); - process.exit(1); -} - -let [sourcePath, resolvedPath, sourceType] = findBundle(program.path); - -// apply RC file -if (!program.norc) { - const rcSettings = rc.readRc([".apigeelintrc"], sourcePath); - if (rcSettings) { - Object.keys(rcSettings) - .filter((key) => key != "path" && key != "list" && !program[key]) - .forEach((key) => { - debug("apigeelint:rc")(`applying [${key}] = ${rcSettings[key]}`); - program[key] = rcSettings[key]; - }); + let [sourcePath, resolvedPath, sourceType] = findBundle(program.path); + + // apply RC file + if (!program.norc) { + const rcSettings = rc.readRc([".apigeelintrc"], sourcePath); + if (rcSettings) { + Object.keys(rcSettings) + .filter((key) => key != "path" && key != "list" && !program[key]) + .forEach((key) => { + debug("apigeelint:rc")(`applying [${key}] = ${rcSettings[key]}`); + program[key] = rcSettings[key]; + }); + } } -} - -const configuration = { - debug: true, - source: { - type: sourceType, - path: resolvedPath, - sourcePath: sourcePath, - bundleType: resolvedPath.endsWith(bundleType.BundleType.SHAREDFLOW) - ? bundleType.BundleType.SHAREDFLOW - : bundleType.BundleType.APIPROXY, - }, - externalPluginsDirectory: program.externalPluginsDirectory, - excluded: {}, - maxWarnings: -1, - profile: "apigee", -}; -if (!isNaN(program.maxWarnings)) { - configuration.maxWarnings = Number.parseInt(program.maxWarnings); -} + const configuration = { + debug: true, + source: { + type: sourceType, + path: resolvedPath, + sourcePath: sourcePath, + bundleType: resolvedPath.endsWith(bundleType.BundleType.SHAREDFLOW) + ? bundleType.BundleType.SHAREDFLOW + : bundleType.BundleType.APIPROXY, + }, + externalPluginsDirectory: program.externalPluginsDirectory, + excluded: {}, + maxWarnings: -1, + profile: "apigee", + }; + + if (!isNaN(program.maxWarnings)) { + configuration.maxWarnings = Number.parseInt(program.maxWarnings); + } -if (program.formatter) { - configuration.formatter = program.formatter || "json.js"; -} + if (program.formatter) { + configuration.formatter = program.formatter || "json.js"; + } -if (program.quiet) { - configuration.output = "none"; -} + if (program.quiet) { + configuration.output = "none"; + } -if (program.ignoreDirectives) { - configuration.ignoreDirectives = true; -} + if (program.ignoreDirectives) { + configuration.ignoreDirectives = true; + } -if (program.excluded && typeof program.excluded === "string") { - configuration.excluded = program.excluded - .split(",") - .map((s) => s.trim()) - .reduce((acc, item) => ((acc[item] = true), acc), {}); -} + if (program.excluded && typeof program.excluded === "string") { + configuration.excluded = program.excluded + .split(",") + .map((s) => s.trim()) + .reduce((acc, item) => ((acc[item] = true), acc), {}); + } -if (program.write) { - configuration.writePath = program.write; -} + if (program.write) { + configuration.writePath = program.write; + } -if (program.profile) { - configuration.profile = program.profile; -} + if (program.profile) { + configuration.profile = program.profile; + } -bl.lint(configuration); + bl.lint(configuration); +})(); diff --git a/lib/package/downloader.js b/lib/package/downloader.js new file mode 100644 index 0000000..11f0e04 --- /dev/null +++ b/lib/package/downloader.js @@ -0,0 +1,110 @@ +/* + Copyright © 2024 Google LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +const fs = require("fs"), + { Readable } = require("stream"), + { finished } = require("stream/promises"), + path = require("path"), + tmp = require("tmp"), + child_process = require("node:child_process"), + debug = require("debug")("apigeelint:download"); + +const downloadBundle = async (downloadSpec) => { + // 0. validate the input. it should be org:ORGNAME,api:APINAME or org:ORGNAME,sf:SHAREDFLOWNAME + let parts = downloadSpec.split(","); + let invalidArgument = () => { + console.log( + "Specify the value in the form org:ORGNAME,api:APINAME or org:ORGNAME,sf:SHAREDFLOWNAME", + ); + process.exit(1); + }; + if (!parts || (parts.length != 2 && parts.length != 3)) { + invalidArgument(); + } + let orgparts = parts[0].split(":"); + if (!orgparts || orgparts.length != 2 || orgparts[0] != "org") { + invalidArgument(); + } + let assetparts = parts[1].split(":"); + if (!assetparts || assetparts.length != 2) { + invalidArgument(); + } + if (assetparts[0] != "api" && assetparts[0] != "sf") { + invalidArgument(); + } + + let providedToken = null; + if (parts.length == 3) { + let tokenParts = parts[2].split(":"); + if (!tokenParts || tokenParts.length != 2 || tokenParts[0] != "token") { + invalidArgument(); + } + providedToken = tokenParts[1]; + } + + const execOptions = { + // cwd: proxyDir, // I think i do not care + encoding: "utf8", + }; + try { + // 1. use the provided token, or get a new one using gcloud. This may fail. + let accessToken = + providedToken || + child_process.execSync("gcloud auth print-access-token", execOptions); + // 2. inquire the revisions + let flavor = assetparts[0] == "api" ? "apis" : "sharedflows"; + const urlbase = `https://apigee.googleapis.com/v1/organizations/${orgparts[1]}/${flavor}`; + const headers = { + Accept: "application/json", + Authorization: `Bearer ${accessToken}`, + }; + + let url = `${urlbase}/${assetparts[1]}/revisions`; + let revisionsResponse = await fetch(url, { method: "GET", headers }); + + // 3. export the latest revision + if (!revisionsResponse.ok) { + throw new Error(`HTTP error: ${revisionsResponse.status}, on GET ${url}`); + } + const revisions = await revisionsResponse.json(); + revisions.sort(); + const rev = revisions[revisions.length - 1]; + url = `${urlbase}/${assetparts[1]}/revisions/${rev}?format=bundle`; + + const tmpdir = tmp.dirSync({ + prefix: `apigeelint-download-${assetparts[0]}`, + keep: false, + }); + const pathToDownloadedAsset = path.join( + tmpdir.name, + `${assetparts[1]}-rev${rev}.zip`, + ); + const stream = fs.createWriteStream(pathToDownloadedAsset); + const { body } = await fetch(url, { method: "GET", headers }); + await finished(Readable.fromWeb(body).pipe(stream)); + return pathToDownloadedAsset; + } catch (ex) { + // Possible causes: No gcloud cli found, or myriad other circumstances. + // Show the error message and first line of stack trace. + console.log(ex.stack.split("\n", 2).join("\n")); + console.log("cannot download the bundle from Apigee. Cannot continue."); + process.exit(1); + } +}; + +module.exports = { + downloadBundle, +}; From 98b05de54a1a5e022174e837db0e6e9c86efa280 Mon Sep 17 00:00:00 2001 From: DinoChiesa Date: Thu, 19 Dec 2024 06:46:12 +0000 Subject: [PATCH 2/2] remove dead code relating to the Edge management server --- cli.js | 4 - lib/package/Bundle.js | 148 ++++++--------------- lib/package/ManagementServer.js | 171 ------------------------ lib/package/processMgmtServer.js | 219 ------------------------------- 4 files changed, 40 insertions(+), 502 deletions(-) delete mode 100644 lib/package/ManagementServer.js delete mode 100644 lib/package/processMgmtServer.js diff --git a/cli.js b/cli.js index 72795e4..59cbd2b 100755 --- a/cli.js +++ b/cli.js @@ -95,10 +95,6 @@ const findBundle = (p) => { "-e, --excluded [value]", "The comma separated list of tests to exclude (default: none)", ) - // .option("-M, --mgmtserver [value]", "Apigee management server") - // .option("-u, --user [value]", "Apigee user account") - // .option("-p, --password [value]", "Apigee password") - // .option("-o, --organization [value]", "Apigee organization") .option( "-x, --externalPluginsDirectory [value]", "Relative or full path to an external plugins directory", diff --git a/lib/package/Bundle.js b/lib/package/Bundle.js index b1296f9..4c7049d 100644 --- a/lib/package/Bundle.js +++ b/lib/package/Bundle.js @@ -39,7 +39,7 @@ function _buildEndpoints(folder, tag, bundle, processFunction) { if (endptFile.endsWith(".xml") && fs.lstatSync(fqFname).isFile()) { const doc = xpath.select( tag, - new Dom().parseFromString(fs.readFileSync(fqFname).toString()) + new Dom().parseFromString(fs.readFileSync(fqFname).toString()), ); doc.forEach((element) => processFunction(element, fqFname, bundle)); } @@ -67,7 +67,12 @@ function buildsharedflows(bundle) { processFunction = function (element, fqFname, bundle) { bundle.proxyEndpoints = bundle.proxyEndpoints || []; bundle.proxyEndpoints.push( - new Endpoint(element, bundle, fqFname, bundleType.BundleType.SHAREDFLOW) + new Endpoint( + element, + bundle, + fqFname, + bundleType.BundleType.SHAREDFLOW, + ), ); }; _buildEndpoints(folder, tag, bundle, processFunction); @@ -92,10 +97,10 @@ function buildPolicies(bundle) { const ext = policyFile.split(".").pop(); if (ext === "xml") { bundle.policies.push( - new Policy(bundle.proxyRoot + "/policies", policyFile, bundle) + new Policy(bundle.proxyRoot + "/policies", policyFile, bundle), ); } - } + }, ); } } @@ -110,7 +115,7 @@ function _buildResources(parent, path, resources) { _buildResources(parent, path + "/" + policyFile, resources); } else if (!policyFile.endsWith("~")) { resources.push( - new Resource(parent, path + "/" + policyFile, policyFile) + new Resource(parent, path + "/" + policyFile, policyFile), ); } }); @@ -132,7 +137,7 @@ Bundle.prototype.getElement = function () { if (!this.element && fs.existsSync(filePath)) { debug(`getElement: filePath:${filePath}`); this.element = new Dom().parseFromString( - fs.readFileSync(filePath).toString() + fs.readFileSync(filePath).toString(), ); } return this.element; @@ -157,7 +162,7 @@ Bundle.prototype.getRevision = function () { try { const attr = xpath.select( `/${this.xPathName}/@revision`, - this.getElement() + this.getElement(), ); this.revision = (attr[0] && attr[0].value) || "undefined"; } catch (e) { @@ -192,7 +197,7 @@ function processFileSystem(config, bundle, cb) { .filter( (file) => fs.lstatSync(path.join(bundle.root, file)).isFile() && - file.endsWith(".xml") + file.endsWith(".xml"), ); if (files.length > 1) { @@ -211,7 +216,7 @@ function processFileSystem(config, bundle, cb) { warningCount: 0, fixableErrorCount: 0, fixableWarningCount: 0, - messages: [] + messages: [], }; //only run for proxy or shared flow root @@ -236,7 +241,7 @@ function processFileSystem(config, bundle, cb) { "No " + bundleTypeName + " folder found in: " + - JSON.stringify(folders) + JSON.stringify(folders), }); } } @@ -260,66 +265,9 @@ function Bundle(config, cb) { this.resolvedPath = config.source.path; this.sourceType = config.source.type; - if (config.source.type === "ManagementServer") { - //shared flow not implemented for management server - if (config.source.bundleType === bundleType.BundleType.SHAREDFLOW) { - throw "SharedFlows for management server not supported"; - } - - const ManagementServer = require("./ManagementServer.js"), - org = config.source.org, - api = config.source.api, - revision = config.source.revision, - tempFolder = "/tmp/" + api, - ms = new ManagementServer({ - org, - authorization: config.source.authorization - }); - - deleteFolderRecursive(tempFolder); - fs.mkdirSync(tempFolder); - const fileStream = fs.createWriteStream(tempFolder + "/apiproxy.zip"); - - ms.get( - "Bundle", - { org }, - { - api, - revision, - onRes: function (res) { - if (res.statusCode === 200) { - res.pipe(fileStream).on("close", function () { - decompress(tempFolder + "/apiproxy.zip", tempFolder).then( - (files) => { - //add the info to the config object here - config.source.path = tempFolder + "/apiproxy"; - processFileSystem(config, bundle, cb); - } - ); - }); - } else { - if (cb) { - cb(null, { - status: res.statusCode, - message: res.statusMessage - }); - } - } - } - }, - function (body) { - //callback on bundle downloaded - if (body.error) { - console.log(body.error); - //bundle.addMessage(JSON.stringify(body.error)); - deleteFolderRecursive(tempFolder); - } - } - ); - } else { - processFileSystem(config, bundle, cb); - } + processFileSystem(config, bundle, cb); } + const isFile = (spec) => { const exists = fs.existsSync(spec), stats = exists && fs.lstatSync(spec); @@ -353,7 +301,7 @@ Bundle.prototype.getReport = function (cb) { ) { try { const docElement = new Dom().parseFromString( - fs.readFileSync(item.filePath).toString() + fs.readFileSync(item.filePath).toString(), ).documentElement; const directives = lintUtil.findDirectives(docElement); d(JSON.stringify(directives, null, 2)); @@ -361,7 +309,7 @@ Bundle.prototype.getReport = function (cb) { const keep = (item) => { d("checking " + JSON.stringify(item)); const directivesForThisRule = directives.filter((directive) => - directive.disable.includes(item.ruleId) + directive.disable.includes(item.ruleId), ); /* * If there is a directive specifying the ruleId for this message @@ -373,21 +321,21 @@ Bundle.prototype.getReport = function (cb) { directivesForThisRule.length == 0 || (item.line && !directivesForThisRule.find( - (d) => d.line + 1 == item.line + (d) => d.line + 1 == item.line, )) || (!item.line && !directivesForThisRule.find( - (d) => d.line == 2 || d.line == 3 + (d) => d.line == 2 || d.line == 3, )) ); }; item.messages = item.messages.filter(keep); // update totals item.warningCount = item.messages.filter( - (m) => m.severity == 1 + (m) => m.severity == 1, ).length; item.errorCount = item.messages.filter( - (m) => m.severity == 2 + (m) => m.severity == 2, ).length; } } catch (e) { @@ -456,7 +404,7 @@ Bundle.prototype.onBundle = function (pluginFunction, cb) { Bundle.prototype.onPolicies = function (pluginFunction, cb) { this.getPolicies().forEach((policy) => - pluginFunction(policy, getcb(`policy '${policy.getName()}'`)) + pluginFunction(policy, getcb(`policy '${policy.getName()}'`)), ); cb(null, {}); }; @@ -472,8 +420,8 @@ Bundle.prototype.onSteps = function (pluginFunction, callback) { proxies.forEach((ep) => ep.onSteps( pluginFunction, - getcb(`STEP proxyendpoint '${ep.getName()}'`) - ) + getcb(`STEP proxyendpoint '${ep.getName()}'`), + ), ); } else { debug("no proxyEndpoints"); @@ -482,8 +430,8 @@ Bundle.prototype.onSteps = function (pluginFunction, callback) { targets.forEach((ep) => ep.onSteps( pluginFunction, - getcb(`STEP targetendpoint '${ep.getName()}'`) - ) + getcb(`STEP targetendpoint '${ep.getName()}'`), + ), ); } else { debug("no targetEndpoints"); @@ -505,16 +453,16 @@ Bundle.prototype.onConditions = function (pluginFunction, callback) { proxies.forEach((ep) => ep.onConditions( pluginFunction, - getcb(`COND proxyendpoint '${ep.getName()}'`) - ) + getcb(`COND proxyendpoint '${ep.getName()}'`), + ), ); } if (targets && targets.length > 0) { targets.forEach((ep) => ep.onConditions( pluginFunction, - getcb(`COND targetendpoint '${ep.getName()}'`) - ) + getcb(`COND targetendpoint '${ep.getName()}'`), + ), ); } } catch (exc1) { @@ -530,7 +478,7 @@ Bundle.prototype.onResources = function (pluginFunction, cb) { bundle .getResources() .forEach((re) => - pluginFunction(re, getcb(`resource '${re.getFileName()}'`)) + pluginFunction(re, getcb(`resource '${re.getFileName()}'`)), ); } cb(null, {}); @@ -539,7 +487,7 @@ Bundle.prototype.onResources = function (pluginFunction, cb) { Bundle.prototype.onFaultRules = function (pluginFunction, cb) { if (this.getFaultRules()) { this.getFaultRules().forEach( - (fr) => fr && pluginFunction(fr, getcb(`faultrule '${fr.getName()}'`)) + (fr) => fr && pluginFunction(fr, getcb(`faultrule '${fr.getName()}'`)), ); } cb(null, {}); @@ -548,7 +496,7 @@ Bundle.prototype.onFaultRules = function (pluginFunction, cb) { Bundle.prototype.onDefaultFaultRules = function (pluginFunction, cb) { if (this.getDefaultFaultRules()) { this.getDefaultFaultRules().forEach( - (dfr) => dfr && pluginFunction(dfr, getcb("defaultfaultrule")) + (dfr) => dfr && pluginFunction(dfr, getcb("defaultfaultrule")), ); } cb(null, {}); @@ -558,7 +506,7 @@ Bundle.prototype.onProxyEndpoints = function (pluginFunction, cb) { const eps = this.getProxyEndpoints(); if (eps && eps.length > 0) { eps.forEach((ep) => - pluginFunction(ep, getcb(`PEP proxyendpoint '${ep.getName()}'`)) + pluginFunction(ep, getcb(`PEP proxyendpoint '${ep.getName()}'`)), ); } cb(null, {}); @@ -568,7 +516,7 @@ Bundle.prototype.onTargetEndpoints = function (pluginFunction, cb) { const eps = this.getTargetEndpoints(); if (eps && eps.length > 0) { eps.forEach((ep) => - pluginFunction(ep, getcb(`TEP targetendpoint '${ep.getName()}'`)) + pluginFunction(ep, getcb(`TEP targetendpoint '${ep.getName()}'`)), ); } cb(null, {}); @@ -629,7 +577,7 @@ Bundle.prototype.getFaultRules = function () { if (!this.faultRules) { this.faultRules = this.getEndpoints().reduce( (a, ep) => [...a, ep.getFaultRules()], - [] + [], ); } return this.faultRules; @@ -639,7 +587,7 @@ Bundle.prototype.getDefaultFaultRules = function () { if (!this.defaultFaultRules) { this.defaultFaultRules = this.getEndpoints().reduce( (a, ep) => [...a, ep.getDefaultFaultRule()], - [] + [], ); } return this.defaultFaultRules; @@ -666,26 +614,10 @@ Bundle.prototype.summarize = function () { policies: this.getPolicies() ? this.getPolicies().map((po) => po.summarize()) - : [] + : [], }; return summary; }; -function deleteFolderRecursive(path) { - if (fs.existsSync(path)) { - fs.readdirSync(path).forEach(function (file) { - const curPath = path + "/" + file; - if (fs.lstatSync(curPath).isDirectory()) { - // recurse - deleteFolderRecursive(curPath); - } else { - // delete file - fs.unlinkSync(curPath); - } - }); - fs.rmdirSync(path); - } -} - //Public module.exports = Bundle; diff --git a/lib/package/ManagementServer.js b/lib/package/ManagementServer.js deleted file mode 100644 index 8c26f7a..0000000 --- a/lib/package/ManagementServer.js +++ /dev/null @@ -1,171 +0,0 @@ -/* - Copyright 2019 Google LLC - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -const uris = { - getEnviros: function(aConfig) { - return "/v1/organizations/" + aConfig.org + "/environments/"; - }, - getOrgAPIs: function(aConfig) { - return "/v1/organizations/" + aConfig.org + "/apis"; - }, - getRevisions: function(aConfig, args) { - return "/v1/organizations/" + aConfig.org + "/apis/" + args; - }, - getBundle: function(aConfig, args) { - return ( - "/v1/o/" + - aConfig.org + - "/apis/" + - args.api + - "/revisions/" + - args.revision + - "?format=bundle" - ); - } - }, - https = require("https"), - os = require("os"); - -ManagementServer.prototype.get = function(uri, aConfig, args, callback) { - aConfig = this.verifyConfig(aConfig); - this.mgmtServerQueue.push({ uri, aConfig, args, callback }); - if (!this.sending) { - makeNextCall(this); - } -}; - -function makeNextCall(ms) { - var call = ms.mgmtServerQueue.shift(); - if (call) { - ms.sending = true; - _get(ms, call.uri, call.aConfig, call.args, function(body, res) { - if (call.callback) { - call.callback(body, res); - makeNextCall(ms); - } - }); - } else { - ms.sending = false; - } -} - -function _get(ms, uri, aConfig, args, callback) { - var data = ""; - - if (aConfig.attempts > 1) { - console.log("processing " + aConfig.attempts + " attempt"); - } - - var options = { - host: aConfig.mgmtApiHost, - port: 443, - path: uris["get" + uri](aConfig, args), - method: "GET", - headers: { - Accept: "application/json", - Authorization: aConfig.authorization - } - }; - - var req = https.request(options, function(res) { - if (res.statusCode === 403) { - callback(data, res); - } else if (res.statusCode >= 300) { - ms.get(uri, aConfig, args, callback); - } - if (args.onRes) { - args.onRes(res, aConfig); - } else { - res.on("data", function(d) { - data += d; - }); - res.on("error", function(e) { - ms.get(uri, aConfig, args, callback); - }); - } - res.on("end", function() { - callback(data, res); - }); - }); - - req.on("error", function(e) { - ms.get(uri, aConfig, args, callback); - }); - req.end(); -} - -ManagementServer.prototype.verifyConfig = function(aConfig) { - aConfig.maxAttempts = aConfig.maxAttempts || this.setup.maxAttempts; - aConfig.attempts = aConfig.attempts || 1; - aConfig.authorization = aConfig.authorization || this.setup.authorization; - aConfig.retryDelay = aConfig.retryDelay || this.setup.retryDelay; - aConfig.mgmtApiHost = aConfig.mgmtApiHost || this.setup.mgmtApiHost; - return aConfig; -}; -ManagementServer.prototype.getAuthorization = function() { - return this.setup.authorization; -}; -ManagementServer.prototype.getHost = function() { - return this.setup.mgmtApiHost; -}; - -function ManagementServer(setup) { - if (typeof setup === "string") { - setup = { org: setup }; - } else { - setup = setup || {}; - } - - if (!setup.user && !setup.password) { - try { - var lintConfig = require(os.homedir() + "/.apigeeLintConfig.json"); - if (lintConfig[setup.org]) { - setup.user = lintConfig[setup.org].user; - setup.password = lintConfig[setup.org].password; - } else if (lintConfig._default) { - setup.user = lintConfig._default.user; - setup.password = lintConfig._default.password; - } - } catch (e) { - //no config init values otherwise - } - } - - this.setup = {}; - this.setup.maxAttempts = setup.maxAttempts || 5; - this.setup.timeout = setup.timeout || 180000; - this.setup.retryDelay = setup.retryDelay || 300; - this.setup.mgmtApiHost = setup.mgmtApiHost || "api.enterprise.apigee.com"; - this.setup.authorization = - setup.authorization || - (setup.user && - setup.password && - "Basic " + - new Buffer(setup.user + ":" + setup.password).toString("base64")) || - (process.env.au && - process.env.as && - "Basic " + - new Buffer(process.env.au + ":" + process.env.as).toString("base64")); - if (!this.setup.authorization) { - throw new Error( - "must provide authorization or .user and .secret or have environment variables au and as" - ); - } - this.mgmtServerQueue = []; - this.sending = false; -} - -module.exports = ManagementServer; diff --git a/lib/package/processMgmtServer.js b/lib/package/processMgmtServer.js deleted file mode 100644 index be5e965..0000000 --- a/lib/package/processMgmtServer.js +++ /dev/null @@ -1,219 +0,0 @@ -/* - Copyright 2019 Google LLC - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -//module to allow for interaction with mgmt server -const decompress = require("decompress"), - fs = require("fs"), - bl = require("./bundleLinter.js"), - ManagementServer = require("./ManagementServer.js"); - -var apis = [], - org = "davidwallen2014", - tempFolder = "./tmp/", - serial = 0, - rootDir = process.cwd(), - completed = {}; - -readCompletedFile(); - -deleteFolderRecursive(tempFolder); -fs.mkdirSync(tempFolder); -var ms = new ManagementServer(org); - -ms.get("OrgAPIs", { org }, [], function(body, res) { - function handleAPI(index) { - var revisions = []; - index = index || 0; - if (index < apis.length) { - var api = apis[index]; - - function handleRevision(revIndex) { - revIndex = revIndex || 0; - if (revIndex < revisions.length) { - var revision = revisions[revIndex]; - if (!isCompleted(ms, org, api, revision)) { - var proxyDir = - tempFolder + org + "-" + api + "-" + revision + "-" + serial++; - fs.mkdirSync(proxyDir); - var fileStream = fs.createWriteStream(proxyDir + "/apiproxy.zip"); - - ms.get( - "Bundle", - { org }, - { - api, - revision, - onRes: function(res, aConfig) { - if (res.statusCode === 200) { - res.pipe(fileStream).on("close", function() { - console.log("starting lint of " + proxyDir); - decompress( - proxyDir + "/apiproxy.zip", - proxyDir - ).then(files => { - bl.lint( - { - source: { - type: "filesystem", - path: proxyDir + "/apiproxy/" - }, - apiUpload: { - destPath: - "https://csdata-test.apigee.net/v1/lintresults", - authorization: ms.getAuthorization(), - organization: aConfig.org - }, - output: function(msg) { - fs.writeFile( - tempFolder + - "/" + - org + - "-" + - api + - "-" + - revision + - ".json", - msg, - function(err) { - msg = null; - if (err) { - return console.log(err); - } - } - ); - msg = null; - } - }, - function() { - addCompleted(ms, org, api, revision); - deleteFolderRecursive(proxyDir); - handleRevision(++revIndex); - } - ); - }); - }); - } - } - }, - function(body, res) { - //callback on bundle downloaded - if (body.error) { - console.log(body); - console.log("timeout bundle download: " + proxyDir); - console.log("cleaning up " + proxyDir); - - fs.writeFile( - tempFolder + - "/" + - org + - "-" + - api + - "-" + - revision + - "-error.json", - body, - function(err) { - if (err) { - return console.log(JSON.stringify(err)); - } - } - ); - console.log("abandonded lint: " + proxyDir); - - deleteFolderRecursive(proxyDir); - handleRevision(++revIndex); - } else { - console.log("completed bundle download: " + proxyDir); - } - } - ); - } else { - handleAPI(++index); - } - } - } - - ms.get("Revisions", { org }, api, function(body, res) { - //callback for recisions - if (body.error) { - console.log(body); - } else { - revisions = JSON.parse(body).revision; - handleRevision(); - } - }); - } - } - //call back once we have OrgAPIs - if (res.statusCode == 403 || res.statusCode == 401) { - throw new Error(res.statusMessage); - } - apis = JSON.parse(body); - handleAPI(); -}); - -function deleteFolderRecursive(path) { - if (fs.existsSync(path)) { - fs.readdirSync(path).forEach(function(file) { - var curPath = path + "/" + file; - if (fs.lstatSync(curPath).isDirectory()) { - // recurse - deleteFolderRecursive(curPath); - } else { - // delete file - fs.unlinkSync(curPath); - } - }); - fs.rmdirSync(path); - } -} - -function getCompletedKey(ms, org, api, revision) { - return ms.getHost() + "-" + org + "-" + api + "-" + revision; -} -function isCompleted(ms, org, api, revision) { - var result = completed[getCompletedKey(ms, org, api, revision)]; - return result || false; -} - -function addCompleted(ms, org, api, revision) { - readCompletedFile(); - completed[getCompletedKey(ms, org, api, revision)] = true; - writeCompletedFile(); -} - -function writeCompletedFile() { - fs.writeFile(rootDir + "/completed.json", JSON.stringify(completed), function( - err - ) { - if (err) { - return console.log(err); - } - }); - return completed; -} - -function readCompletedFile() { - try { - completed = JSON.parse( - fs.readFileSync(rootDir + "/completed.json", "utf8") - ); - } catch (e) { - console.log("initing completed file"); - completed = {}; - writeCompletedFile(); - } -}