From 1d602c4430bd17bc8f5ed3cd1b85e6a58ceff86f Mon Sep 17 00:00:00 2001 From: DinoChiesa Date: Thu, 19 Dec 2024 04:59:00 +0000 Subject: [PATCH 1/4] chore: code cleanup. add new test for PO039. --- PO039-test.js | 105 ++++++++++++ .../EX-001-CheckForPoliciesWhileStreaming.js | 53 ------ ...EX-PO001-CheckForPoliciesWhileStreaming.js | 87 ++++++++++ lib/package/Endpoint.js | 1 - lib/package/HTTPProxyConnection.js | 58 +++---- lib/package/Policy.js | 6 +- lib/package/bundleLinter.js | 159 +++++++++--------- lib/package/plugins/PD004-proxyNameAttr.js | 2 +- .../plugins/PO006-checkPolicyNameAttribute.js | 4 +- .../PO039-messageLogging-responseType.js | 2 +- .../resources/PO039/ML-test-invalid4.xml | 18 ++ test/specs/testBundle.js | 2 +- 12 files changed, 326 insertions(+), 171 deletions(-) create mode 100644 PO039-test.js delete mode 100644 externalPlugins/EX-001-CheckForPoliciesWhileStreaming.js create mode 100644 externalPlugins/EX-PO001-CheckForPoliciesWhileStreaming.js create mode 100644 test/fixtures/resources/PO039/ML-test-invalid4.xml diff --git a/PO039-test.js b/PO039-test.js new file mode 100644 index 0000000..67c0961 --- /dev/null +++ b/PO039-test.js @@ -0,0 +1,105 @@ +/* + Copyright 2019-2024 Google LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +const testID = "PO039", + assert = require("assert"), + fs = require("fs"), + path = require("path"), + bl = require("../../lib/package/bundleLinter.js"), + plugin = require(bl.resolvePlugin(testID)), + debug = require("debug")("apigeelint:" + testID), + Policy = require("../../lib/package/Policy.js"), + Dom = require("@xmldom/xmldom").DOMParser, + rootDir = path.resolve(__dirname, "../fixtures/resources/PO039"); + +const test = (suffix, cb) => { + const filename = `ML-test-${suffix}.xml`; + it(`should correctly process ${filename}`, () => { + const fqfname = path.join(rootDir, filename), + policyXml = fs.readFileSync(fqfname, "utf-8"), + doc = new Dom().parseFromString(policyXml), + p = new Policy(rootDir, filename, this, doc); + + p.getElement = () => doc.documentElement; + + //plugin.onBundle({ profile: "apigee" }); + + plugin.onPolicy(p, (e, foundIssues) => { + assert.equal(e, undefined, "should be undefined"); + cb(p, foundIssues); + }); + }); +}; + +describe(`${testID} - MessageLogging RessourceType element`, () => { + // test all the valid cases + fs.readdirSync(rootDir) + .map((shortFileName) => { + let m = shortFileName.match("^.+-(valid.+)\\.xml$"); + if (m) { + return m[1]; + } + }) + .filter((suffix) => suffix) + .forEach((suffix) => { + test(suffix, (p, foundIssues) => { + const messages = p.getReport().messages; + assert.ok(messages, "messages undefined"); + debug(messages); + assert.equal(foundIssues, false); + }); + }); + + test("invalid1", (p, foundIssues) => { + assert.equal(foundIssues, true); + const messages = p.getReport().messages; + assert.ok(messages, "messages undefined"); + debug(messages); + assert.equal(messages.length, 1, "unexpected number of messages"); + assert.ok(messages[0].message, "did not find message 0"); + assert.equal( + messages[0].message, + "The value 'gce_instance' should not be used here. ResourceType should be 'api'", + ); + }); + + test("invalid2", (p, foundIssues) => { + assert.equal(foundIssues, true); + const messages = p.getReport().messages; + assert.ok(messages, "messages undefined"); + debug(messages); + assert.equal(messages.length, 1, "unexpected number of messages"); + assert.ok(messages[0].message, "did not find message 0"); + assert.equal( + messages[0].message, + "The value 'apigee.googleapis.com/Environment' should not be used here. ResourceType should be 'api'", + ); + }); + + test("invalid3", (p, foundIssues) => { + assert.equal(foundIssues, true); + const messages = p.getReport().messages; + assert.ok(messages, "messages undefined"); + debug(messages); + assert.equal(messages.length, 2, "unexpected number of messages"); + assert.ok(messages[0].message, "did not find message 0"); + assert.equal(messages[0].message, "Unsupported element 'NotKey'"); + assert.equal( + messages[1].message, + "Label is missing a required Element: Key.", + ); + }); +}); diff --git a/externalPlugins/EX-001-CheckForPoliciesWhileStreaming.js b/externalPlugins/EX-001-CheckForPoliciesWhileStreaming.js deleted file mode 100644 index ce05e4d..0000000 --- a/externalPlugins/EX-001-CheckForPoliciesWhileStreaming.js +++ /dev/null @@ -1,53 +0,0 @@ -const plugin = { - ruleId: "EX-001", - name: "Streaming", - message: "Check for policies while streaming is enabled", - fatal: false, - severity: 2, // 1 = warn, 2 = error - nodeType: "Bundle", - enabled: true -}; - -const onBundle = function(bundle, cb) { - let hadWarnErr=false, isProxyStreamingEnabled=false, isTargetStreamingEnabled=false; - const proxies = bundle.getProxyEndpoints(); - proxies.forEach((proxyEndpoint, _p) => { - const httpProxyConnection = proxyEndpoint.getHTTPProxyConnection(); - if(httpProxyConnection){ - let properties = httpProxyConnection.getProperties(); - if(properties!=null && (properties["request.streaming.enabled"]=="true" || properties["response.streaming.enabled"]=="true")){ - isProxyStreamingEnabled = true; - } - } - }); - const targets = bundle.getTargetEndpoints(); - targets.forEach((targetEndpoint, _t) => { - const httpTargetConnection = targetEndpoint.getHTTPTargetConnection(); - if(httpTargetConnection){ - let properties = httpTargetConnection.getProperties(); - if(properties!=null && (properties["request.streaming.enabled"]=="true" || properties["response.streaming.enabled"]=="true")){ - isTargetStreamingEnabled = true; - } - } - }); - - if(isProxyStreamingEnabled || isTargetStreamingEnabled){ - bundle.getPolicies().forEach(function(policy) { - if ((policy.getType() === "AssignMessage" || policy.getType() === "ExtractVariables") && policy.getSteps().length > 0) { - bundle.addMessage({ - plugin, - source: policy.getSource(), - line: policy.getElement().lineNumber, - column: policy.getElement().columnNumber, - message: "ExtractVariables/AssignMessage policies not allowed when streaming is enabled" - }); - hadWarnErr = true; - } - }); - } -}; - -module.exports = { - plugin, - onBundle -}; \ No newline at end of file diff --git a/externalPlugins/EX-PO001-CheckForPoliciesWhileStreaming.js b/externalPlugins/EX-PO001-CheckForPoliciesWhileStreaming.js new file mode 100644 index 0000000..4658fd0 --- /dev/null +++ b/externalPlugins/EX-PO001-CheckForPoliciesWhileStreaming.js @@ -0,0 +1,87 @@ +/* + Copyright 2019-2024 Google LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +const plugin = { + ruleId: "EX-PO001", + name: "Streaming", + message: "Check for policies while streaming is enabled", + fatal: false, + severity: 2, // 1 = warn, 2 = error + nodeType: "Bundle", + enabled: true, +}; + +const onBundle = function (bundle, cb) { + let flagged = false, + isProxyStreamingEnabled = false, + isTargetStreamingEnabled = false; + const proxies = bundle.getProxyEndpoints(); + proxies.forEach((proxyEndpoint, _p) => { + const httpProxyConnection = proxyEndpoint.getHTTPProxyConnection(); + if (httpProxyConnection) { + let properties = httpProxyConnection.getProperties(); + if ( + properties && + (properties["request.streaming.enabled"] == "true" || + properties["response.streaming.enabled"] == "true") + ) { + isProxyStreamingEnabled = true; + } + } + }); + const targets = bundle.getTargetEndpoints(); + targets.forEach((targetEndpoint, _t) => { + const httpTargetConnection = targetEndpoint.getHTTPTargetConnection(); + if (httpTargetConnection) { + let properties = httpTargetConnection.getProperties(); + if ( + properties && + (properties["request.streaming.enabled"] == "true" || + properties["response.streaming.enabled"] == "true") + ) { + isTargetStreamingEnabled = true; + } + } + }); + + if (isProxyStreamingEnabled || isTargetStreamingEnabled) { + bundle.getPolicies().forEach(function (policy) { + if ( + (policy.getType() === "AssignMessage" || + policy.getType() === "ExtractVariables") && + policy.getSteps().length > 0 + ) { + bundle.addMessage({ + plugin, + source: policy.getSource(), + line: policy.getElement().lineNumber, + column: policy.getElement().columnNumber, + message: + "ExtractVariables/AssignMessage policies not allowed when streaming is enabled", + }); + flagged = true; + } + }); + } + if (typeof cb == "function") { + cb(null, flagged); + } +}; + +module.exports = { + plugin, + onBundle, +}; diff --git a/lib/package/Endpoint.js b/lib/package/Endpoint.js index 269545b..b86d3e0 100644 --- a/lib/package/Endpoint.js +++ b/lib/package/Endpoint.js @@ -41,7 +41,6 @@ function Endpoint(element, bundle, fname, bundletype) { this.httpProxyConnection = null; this.httpTargetConnection = null; this.report = { - //filePath: path.join(bundle.sourcePath, path.basename(fname)), filePath: lintUtil.effectivePath(bundle, fname), errorCount: 0, warningCount: 0, diff --git a/lib/package/HTTPProxyConnection.js b/lib/package/HTTPProxyConnection.js index bb8f859..238a552 100644 --- a/lib/package/HTTPProxyConnection.js +++ b/lib/package/HTTPProxyConnection.js @@ -1,5 +1,5 @@ /* - Copyright 2019 Google LLC + Copyright 2019,2024 Google LLC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ function HTTPProxyConnection(element, parent) { this.element = element; } -HTTPProxyConnection.prototype.getName = function() { +HTTPProxyConnection.prototype.getName = function () { if (!this.name) { var attr = xpath.select("//@name", this.element); this.name = (attr[0] && attr[0].value) || ""; @@ -29,15 +29,15 @@ HTTPProxyConnection.prototype.getName = function() { return this.name; }; -HTTPProxyConnection.prototype.getMessages = function() { +HTTPProxyConnection.prototype.getMessages = function () { return this.parent.getMessages(); }; -HTTPProxyConnection.prototype.getLines = function(start, stop) { +HTTPProxyConnection.prototype.getLines = function (start, stop) { return this.parent.getLines(start, stop); }; -HTTPProxyConnection.prototype.getSource = function() { +HTTPProxyConnection.prototype.getSource = function () { if (!this.source) { var start = this.element.lineNumber - 1, stop = this.element.nextSibling.lineNumber - 1; @@ -46,57 +46,57 @@ HTTPProxyConnection.prototype.getSource = function() { return this.source; }; -HTTPProxyConnection.prototype.getType = function() { +HTTPProxyConnection.prototype.getType = function () { return this.element.tagName; }; -HTTPProxyConnection.prototype.getBasePath = function() { +HTTPProxyConnection.prototype.getBasePath = function () { if (!this.basePath) { - //find the preflow tag var doc = xpath.select("./BasePath", this.element); if (doc && doc[0]) { - this.basePath = - (doc && doc[0] && doc[0].childNodes[0].nodeValue) || ""; + this.basePath = (doc && doc[0] && doc[0].childNodes[0].nodeValue) || ""; } } return this.basePath; }; - - -HTTPProxyConnection.prototype.getElement = function() { +HTTPProxyConnection.prototype.getElement = function () { return this.element; }; -HTTPProxyConnection.prototype.getParent = function() { +HTTPProxyConnection.prototype.getParent = function () { return this.parent; }; -HTTPProxyConnection.prototype.addMessage = function(msg) { +HTTPProxyConnection.prototype.addMessage = function (msg) { if (!msg.hasOwnProperty("entity")) { msg.entity = this; } this.parent.addMessage(msg); }; -HTTPProxyConnection.prototype.getProperties = function() { - var props = new Map(); - if (!this.properties) { - var propsNodeList = xpath.select("./Properties", this.element)[0].childNodes; - Array.from(propsNodeList).forEach(function(prop) { - if (prop.childNodes){ - props[prop.attributes[0].nodeValue]=prop.childNodes[0].nodeValue; +HTTPProxyConnection.prototype.getProperties = function () { + if (this.properties == undefined) { + let props = {}; + this.properties = props; + let propsNodeList = xpath.select("./Properties", this.element); + if (propsNodeList && propsNodeList[0]) { + Array.from(propsNodeList[0].childNodes).forEach((prop) => { + if (prop.childNodes && prop.attributes[0] && prop.childNodes[0]) { + props[prop.attributes[0].nodeValue] = prop.childNodes[0].nodeValue; } - }); + }); + } } - return props; + return this.properties; }; -HTTPProxyConnection.prototype.summarize = function() { - var summary = {}; - summary.name = this.getName(); - summary.basePath = this.getBasePath(); - return summary; +HTTPProxyConnection.prototype.summarize = function () { + // not sure this is ever used + return { + name: this.getName(), + basePath: this.getBasePath(), + }; }; //Public diff --git a/lib/package/Policy.js b/lib/package/Policy.js index a5a458a..7a5ac50 100644 --- a/lib/package/Policy.js +++ b/lib/package/Policy.js @@ -37,12 +37,12 @@ function Policy(realpath, fname, bundle, doc) { fixableWarningCount: 0, messages: [], }; + this.name = null; } Policy.prototype.getName = function () { - if (!this.name) { - let attr = xpath.select("//@name", this.getElement()); - this.name = (attr[0] && attr[0].value) || ""; + if (this.name == null) { + this.name = xpath.select("string(/*/@name)", this.getElement()); } return this.name; }; diff --git a/lib/package/bundleLinter.js b/lib/package/bundleLinter.js index 538e4e1..ad2908c 100644 --- a/lib/package/bundleLinter.js +++ b/lib/package/bundleLinter.js @@ -57,17 +57,55 @@ function exportReport(providedPath, stringifiedReport) { }); } -const getPluginPath = () => path.resolve(path.join(__dirname, "plugins")); +const internalPluginIdRe = new RegExp("^[A-Z]{2}[0-9]{3}$"); +const internalPluginFileRe = new RegExp("^[A-Z]{2}[0-9]{3}-.+\\.js$"); +const externalPluginIdRe = new RegExp("^EX-[A-Z]{2}[0-9]{3}$"); +const externalPluginFileRe = new RegExp("^EX-[A-Z]{2}[0-9]{3}-.+\\.js$"); +const nonlibrarySourceRe = new RegExp("^_.+\\.js$"); -const listPlugins = () => fs.readdirSync(getPluginPath()).filter(resolvePlugin); +/* exposed for testing */ +const getPluginResolver = (d, isInternal) => (idOrFilename) => { + if (idOrFilename.endsWith(".js")) { + if (idOrFilename.indexOf("/") < 0) { + if (!nonlibrarySourceRe.test(idOrFilename)) { + debug(`resolvePlugin file(${idOrFilename}) , prepending path...`); + return path.resolve(d, idOrFilename); + } else return null; + } else { + return idOrFilename; + } + } else { + let re = isInternal ? internalPluginIdRe : externalPluginIdRe; + if (re.test(idOrFilename)) { + let p = fs + .readdirSync(path.resolve(d)) + .filter((p) => p.startsWith(idOrFilename) && p.endsWith(".js")); + if (p.length > 1) { + throw new Error("plugin conflict: " + JSON.stringify(p)); + } + return p.length ? path.resolve(d, p[0]) : null; + } + } + return null; +}; -const listRuleIds = () => listPlugins().map((s) => s.substring(0, 5)); +const getInternalPluginsPath = () => + path.resolve(path.join(__dirname, "plugins")); + +const resolveInternalPlugin = getPluginResolver(getInternalPluginsPath(), true); -const listExternalRuleIds = (externalDir) => +const listPlugins = () => fs - .readdirSync(externalDir) - .filter(resolvePlugin) - .map((s) => s.substring(0, 8)); + .readdirSync(getInternalPluginsPath()) + .filter((fname) => internalPluginFileRe.test(fname)); + +const listExternalPlugins = (d) => + fs.readdirSync(d).filter((fname) => externalPluginFileRe.test(fname)); + +const listRuleIds = () => listPlugins().map((s) => s.substring(0, 5)); + +const listExternalRuleIds = (d) => + listExternalPlugins(d).map((s) => s.substring(0, 8)); const listFormatters = () => fs @@ -75,37 +113,30 @@ const listFormatters = () => .filter((s) => s.endsWith(".js")); const lint = function (config, done) { - return new Bundle(config, function (bundle, err) { - if (err) { - return done ? done(null, err) : console.log(err); + return new Bundle(config, function (bundle, error) { + if (error) { + return done ? done(null, error) : console.log(error); } - // for each builtin plugin - const normalizedPath = getPluginPath(); - fs.readdirSync(normalizedPath).forEach(function (file) { - if (!config.plugins || contains(config.plugins, file)) { - try { - executePlugin(normalizedPath + "/" + file, bundle); - } catch (e) { - debug("plugin error: " + file + " " + e); - } - } - }); - - // for each external plugin - if (config.externalPluginsDirectory) { - fs.readdirSync(config.externalPluginsDirectory).forEach(function (file) { - if (!config.plugins || contains(config.plugins, file)) { + const executePlugins = (list, pluginsPath) => { + list.forEach(function (filename) { + if (!config.plugins || contains(config.plugins, filename)) { try { - executePlugin( - path.resolve(config.externalPluginsDirectory) + "/" + file, - bundle - ); + executePlugin(path.resolve(pluginsPath, filename), bundle); } catch (e) { - debug("plugin error: " + file + " " + e); + console.error("plugin error: " + filename + " " + e.stack); } } }); + }; + + executePlugins(listPlugins(), getInternalPluginsPath()); + + if (config.externalPluginsDirectory) { + executePlugins( + listExternalPlugins(config.externalPluginsDirectory), + config.externalPluginsDirectory, + ); } const formatter = config.formatter || "json.js", @@ -124,31 +155,25 @@ const lint = function (config, done) { exportReport(config.writePath, formattedReport); } - if (done) { - done(bundle, null); - } - - // Exit code should return 1 when there are errors + // Exit code should return 1 when there are errors or too many warnings if (typeof config.setExitCode == "undefined" || config.setExitCode) { - bundle.getReport().some(function (value) { - if (value.errorCount > 0) { - process.exitCode = 1; - return; - } - }); + process.exitCode = bundle + .getReport() + .find((value) => value.errorCount > 0) + ? 1 + : 0; - // Exit code should return 1 when more than maximum number of warnings allowed - if (config.maxWarnings >= 0) { - let warningCount = 0; - bundle + if (!process.exitCode && config.maxWarnings >= 0) { + let warningCount = bundle .getReport() - .forEach((report) => (warningCount += report.warningCount)); + .reduce((a, item) => a + item.warningCount, 0); if (warningCount > config.maxWarnings) { process.exitCode = 1; - return; } } } + + return done ? done(bundle, null) : null; }); }; @@ -172,7 +197,7 @@ var getFormatter = function (format) { } catch (ex) { throw new Error( `There was a problem loading formatter: ${formatterPath}`, - { cause: ex } + { cause: ex }, ); } } else { @@ -183,34 +208,9 @@ var getFormatter = function (format) { const bfnName = (term) => term == "Bundle" ? "onBundle" : pluralize("on" + term, 2); -const pluginIdRe1 = new RegExp("^[A-Z]{2}[0-9]{3}$"); -const pluginIdRe2 = new RegExp("^_.+.js$"); - -/* exposed for testing */ -const resolvePlugin = (idOrFilename) => { - if (idOrFilename.endsWith(".js")) { - if (idOrFilename.indexOf("/") < 0) { - if (!pluginIdRe2.test(idOrFilename)) { - debug(`resolvePlugin file(${idOrFilename}) , prepending path...`); - return path.resolve(getPluginPath(), idOrFilename); - } else return null; - } else { - return idOrFilename; - } - } else if (pluginIdRe1.test(idOrFilename)) { - let p = fs - .readdirSync(path.resolve(getPluginPath())) - .filter((p) => p.startsWith(idOrFilename) && p.endsWith(".js")); - if (p.length > 1) { - throw new Error("plugin conflict: " + JSON.stringify(p)); - } - return p.length ? path.resolve(getPluginPath(), p[0]) : null; - } - return null; -}; - const executePlugin = function (file, bundle) { - let pluginPath = resolvePlugin(file); + const stats = fs.existsSync(file) && fs.statSync(file); + let pluginPath = stats && stats.isFile() ? file : resolveInternalPlugin(file); if (pluginPath) { debug(`executePlugin file(${pluginPath})`); @@ -230,7 +230,7 @@ const executePlugin = function (file, bundle) { "Resource", "Policy", "FaultRule", - "DefaultFaultRule" + "DefaultFaultRule", ]; entityTypes.forEach((etype) => { @@ -247,12 +247,11 @@ const executePlugin = function (file, bundle) { module.exports = { lint, - getPluginPath, listPlugins, listRuleIds, listExternalRuleIds, listFormatters, executePlugin, - resolvePlugin, // for testing - getFormatter + resolvePlugin: resolveInternalPlugin, // for testing + getFormatter, }; diff --git a/lib/package/plugins/PD004-proxyNameAttr.js b/lib/package/plugins/PD004-proxyNameAttr.js index 40c5337..23acccd 100644 --- a/lib/package/plugins/PD004-proxyNameAttr.js +++ b/lib/package/plugins/PD004-proxyNameAttr.js @@ -48,7 +48,7 @@ const onProxyEndpoint = function (endpoint, cb) { }); flagged = true; } else if (basename !== nameFromAttribute) { - const nameAttr = endpoint.select("//@name"); + const nameAttr = endpoint.select("/*/@name"); endpoint.addMessage({ plugin, line: nameAttr[0].lineNumber, diff --git a/lib/package/plugins/PO006-checkPolicyNameAttribute.js b/lib/package/plugins/PO006-checkPolicyNameAttribute.js index 3f38959..f1574f5 100644 --- a/lib/package/plugins/PO006-checkPolicyNameAttribute.js +++ b/lib/package/plugins/PO006-checkPolicyNameAttribute.js @@ -1,5 +1,5 @@ /* - Copyright 2019-2020 Google LLC + Copyright 2019-2020,2024 Google LLC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -32,7 +32,7 @@ const onPolicy = function(policy, cb) { flagged = false; if (shortFilename !== nameFromAttribute) { - let nameAttr = policy.select('//@name'); + let nameAttr = policy.select('/*/@name'); policy.addMessage({ plugin, line: nameAttr[0].lineNumber, diff --git a/lib/package/plugins/PO039-messageLogging-responseType.js b/lib/package/plugins/PO039-messageLogging-responseType.js index 4aec269..00898db 100644 --- a/lib/package/plugins/PO039-messageLogging-responseType.js +++ b/lib/package/plugins/PO039-messageLogging-responseType.js @@ -40,7 +40,7 @@ const onPolicy = function (policy, cb) { ); try { if (clElements && clElements[0]) { - debug(`found ${clElements.length} response element(s)`); + debug(`found ${clElements.length} CloudLogging element(s)`); if (clElements[1]) { hadWarning = true; policy.addMessage({ diff --git a/test/fixtures/resources/PO039/ML-test-invalid4.xml b/test/fixtures/resources/PO039/ML-test-invalid4.xml new file mode 100644 index 0000000..73a85aa --- /dev/null +++ b/test/fixtures/resources/PO039/ML-test-invalid4.xml @@ -0,0 +1,18 @@ + + + projects/{organization.name}/logs/{proxylog} + { "foo" : "bar"} + + + projects/{organization.name}/logs/{proxylog} + { + "organization": "{organization.name}", + "environment": "{environment.name}", + "apiproxy": "{apiproxy.name}", + "revision": "{apiproxy.revision}", + "messageid": "{messageid}", + "url": "{proxy.url}" + } + + + diff --git a/test/specs/testBundle.js b/test/specs/testBundle.js index dd86a9a..5915f1a 100644 --- a/test/specs/testBundle.js +++ b/test/specs/testBundle.js @@ -49,7 +49,7 @@ describe("addMessage", function () { bundle.addMessage({ plugin, message }); bundle.getReport((report) => { - console.log(JSON.stringify(report, null, 2)); + //console.log(JSON.stringify(report, null, 2)); let bundleResult = report.find( (element) => element.filePath === proxyPath, From 26bb956a5ec9d9a05a5393343d2eb2f71e6be679 Mon Sep 17 00:00:00 2001 From: DinoChiesa Date: Thu, 19 Dec 2024 06:33:38 +0000 Subject: [PATCH 2/4] insert missing copyright into apigeelintrc.js --- lib/package/apigeelintrc.js | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/package/apigeelintrc.js b/lib/package/apigeelintrc.js index 3d76074..4e14382 100644 --- a/lib/package/apigeelintrc.js +++ b/lib/package/apigeelintrc.js @@ -1,11 +1,18 @@ -// apigeelintrc.js -// ------------------------------------------------------------------ -// -// created: Mon Apr 15 18:14:54 2024 -// last saved: <2024-July-19 12:24:12> +/* + Copyright © 2023-2024 Google LLC -/* jshint esversion:9, node:true, strict:implied */ -/* global process, console, Buffer */ + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ const os = require("os"); const fs = require("fs"); @@ -32,7 +39,7 @@ const findRc = (settingsfiles, sourcedir) => { } return combine([sourcedir, ...locations], settingsfiles).find( - (candidate) => fs.existsSync(candidate) && fs.statSync(candidate).isFile() + (candidate) => fs.existsSync(candidate) && fs.statSync(candidate).isFile(), ); }; @@ -57,5 +64,5 @@ const readRc = (settingsfiles, sourcedir) => { module.exports = { readRc, - findRc + findRc, }; From b736fb80a528f78abc2c597bbc55da5dff9d65b8 Mon Sep 17 00:00:00 2001 From: DinoChiesa Date: Thu, 19 Dec 2024 06:30:52 +0000 Subject: [PATCH 3/4] feat: implement the download feature --- README.md | 41 +++++- cli.js | 257 +++++++++++++++++++++----------------- lib/package/downloader.js | 110 ++++++++++++++++ 3 files changed, 290 insertions(+), 118 deletions(-) create mode 100644 lib/package/downloader.js diff --git a/README.md b/README.md index bb4d0d1..f587e24 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ Usage: apigeelint [options] Options: -V, --version output the version number -s, --path Path of the proxy to analyze + -d, --download [value] Download the API proxy or sharedflow to analyze. Exclusive of -s / --path. Example: org:ORG,API:proxyname or org:ORG,sf:SHAREDFLOWNAME -f, --formatter [value] Specify formatters (default: json.js) -w, --write [value] file path to write results -e, --excluded [value] The comma separated list of tests to exclude (default: none) @@ -106,10 +107,48 @@ archive. This tool also can read and analyze these zipped bundles: apigeelint -f table.js -s path/to/your/apiproxy.zip ``` -The tool will unzip the bundle to a temporary directory, perform the analysis, +The tool will unzip the bundle into a temporary directory, perform the analysis, and then remove the temporary directory. +### Basic usage: downloading a proxy bundle to analyze + +You can ask apigeelint to export an API Proxy or Sharedflow bundle from Apigee, +and analyze the resulting zip archive. This connects to apigee.googleapis.com to +perform the export, which means it will work only with Apigee X or hybrid. + +``` +# to download and then analyze a proxy bundle +apigeelint -f table.js -d org:your-org-name,api:name-of-your-api-proxy + +# to download and then analyze a sharedflow bundle +apigeelint -f table.js -d org:your-org-name,sf:name-of-your-shared-flow +``` + +With this invocation, the tool will: +- obtain a token using the `gcloud auth print-access-token` command +- use the token to inquire the latest revision of the proxy or sharedflow +- use the token to download the bundle for the latest revision +- unzip the bundle into a temporary directory +- perform the lint analysis +- render the result +- and then remove the temporary directory + +If you do not have the [`gcloud` command line +tool](https://cloud.google.com/sdk/gcloud) installed, and available on your +path, this will fail. + + +You can also specify a token you have obtained previously: + +``` +apigeelint -f table.js -d org:your-org-name,api:name-of-your-api-proxy,token:ACCESS_TOKEN_HERE +``` + +In this case, apigeelint does not try to use `gcloud` to obtain an access token. + + + ### Using External Plugins: ``` apigeelint -x ./externalPlugins -s path/to/your/apiproxy -f table.js diff --git a/cli.js b/cli.js index a75a1a7..72795e4 100755 --- a/cli.js +++ b/cli.js @@ -24,6 +24,7 @@ const program = require("commander"), bl = require("./lib/package/bundleLinter.js"), rc = require("./lib/package/apigeelintrc.js"), pkj = require("./package.json"), + downloader = require("./lib/package/downloader.js"), bundleType = require("./lib/package/BundleTypes.js"), debug = require("debug"); @@ -77,136 +78,158 @@ const findBundle = (p) => { ); }; -program - .version(pkj.version) - .option( - "-s, --path ", - "Path of the exploded apiproxy or sharedflowbundle directory", - ) - .option("-f, --formatter [value]", "Specify formatters (default: json.js)") - .option("-w, --write [value]", "file path to write results") - .option( - "-e, --excluded [value]", - "The comma separated list of tests to exclude (default: none)", - ) - // .option("-M, --mgmtserver [value]", "Apigee management server") - // .option("-u, --user [value]", "Apigee user account") - // .option("-p, --password [value]", "Apigee password") - // .option("-o, --organization [value]", "Apigee organization") - .option( - "-x, --externalPluginsDirectory [value]", - "Relative or full path to an external plugins directory", - ) - .option( - "-q, --quiet", - "do not emit the report to stdout. (can use --write option to write to file)", - ) - .option( - "--list", - "do not execute, instead list the available plugins and formatters", - ) - .option( - "--maxWarnings [value]", - "Number of warnings to trigger nonzero exit code (default: -1)", - ) - .option("--profile [value]", "Either apigee or apigeex (default: apigee)") - .option( - "--norc", - "do not search for and use the .apigeelintrc file for settings", - ) - .option( - "--ignoreDirectives", - "ignore any directives within XML files that disable warnings", - ); +(async function main() { + program + .version(pkj.version) + .option( + "-s, --path ", + "Path of the exploded apiproxy or sharedflowbundle directory", + ) + .option( + "-d, --download [value]", + "Download the API proxy or sharedflow to analyze. Exclusive of -s / --path. Example: org:ORG,API:proxyname or org:ORG,sf:SHAREDFLOWNAME", + ) + .option("-f, --formatter [value]", "Specify formatters (default: json.js)") + .option("-w, --write [value]", "file path to write results") + .option( + "-e, --excluded [value]", + "The comma separated list of tests to exclude (default: none)", + ) + // .option("-M, --mgmtserver [value]", "Apigee management server") + // .option("-u, --user [value]", "Apigee user account") + // .option("-p, --password [value]", "Apigee password") + // .option("-o, --organization [value]", "Apigee organization") + .option( + "-x, --externalPluginsDirectory [value]", + "Relative or full path to an external plugins directory", + ) + .option( + "-q, --quiet", + "do not emit the report to stdout. (can use --write option to write to file)", + ) + .option( + "--list", + "do not scan, instead list the available plugins and formatters", + ) + .option( + "--maxWarnings [value]", + "Number of warnings to trigger nonzero exit code (default: -1)", + ) + .option("--profile [value]", "Either apigee or apigeex (default: apigee)") + .option( + "--norc", + "do not search for and use the .apigeelintrc file for settings", + ) + .option( + "--ignoreDirectives", + "ignore any directives within XML files that disable warnings", + ); -program.on("--help", function () { - console.log("\nExample: apigeelint -f table.js -s sampleProxy/apiproxy"); - console.log(""); -}); + program.on("--help", function () { + console.log("\nExamples:"); + console.log(" apigeelint -f table.js -s sampleProxy/apiproxy\n"); + console.log(" apigeelint -f table.js -s path/to/your/apiproxy.zip\n"); + console.log( + " apigeelint -f table.js --download org:my-org,api:my-proxy\n", + ); + console.log(""); + }); + + program.parse(process.argv); + + if (program.list) { + console.log("available plugins: " + bl.listRuleIds().join(", ") + "\n"); + console.log("available formatters: " + bl.listFormatters().join(", ")); + if (fs.existsSync(program.externalPluginsDirectory)) { + console.log( + "\n" + + "available external plugins: " + + bl.listExternalRuleIds(program.externalPluginsDirectory).join(", ") + + "\n", + ); + } + process.exit(0); + } -program.parse(process.argv); + if (program.download) { + if (program.path) { + console.log( + "you must specify either the -s /--path option, or the -d /--download option. Not both.", + ); + process.exit(1); + } + // This will work only with Apigee X/hybrid + program.path = await downloader.downloadBundle(program.download); + } -if (program.list) { - console.log("available plugins: " + bl.listRuleIds().join(", ") + "\n"); - console.log("available formatters: " + bl.listFormatters().join(", ")); - if (fs.existsSync(program.externalPluginsDirectory)) { + if (!program.path) { console.log( - "\n" + - "available external plugins: " + - bl.listExternalRuleIds(program.externalPluginsDirectory).join(", ") + - "\n", + "you must specify the -s option, or the long form of that: --path ", ); + process.exit(1); } - process.exit(0); -} -if (!program.path) { - console.log( - "you must specify the -s option, or the long form of that: --path ", - ); - process.exit(1); -} - -let [sourcePath, resolvedPath, sourceType] = findBundle(program.path); - -// apply RC file -if (!program.norc) { - const rcSettings = rc.readRc([".apigeelintrc"], sourcePath); - if (rcSettings) { - Object.keys(rcSettings) - .filter((key) => key != "path" && key != "list" && !program[key]) - .forEach((key) => { - debug("apigeelint:rc")(`applying [${key}] = ${rcSettings[key]}`); - program[key] = rcSettings[key]; - }); + let [sourcePath, resolvedPath, sourceType] = findBundle(program.path); + + // apply RC file + if (!program.norc) { + const rcSettings = rc.readRc([".apigeelintrc"], sourcePath); + if (rcSettings) { + Object.keys(rcSettings) + .filter((key) => key != "path" && key != "list" && !program[key]) + .forEach((key) => { + debug("apigeelint:rc")(`applying [${key}] = ${rcSettings[key]}`); + program[key] = rcSettings[key]; + }); + } } -} - -const configuration = { - debug: true, - source: { - type: sourceType, - path: resolvedPath, - sourcePath: sourcePath, - bundleType: resolvedPath.endsWith(bundleType.BundleType.SHAREDFLOW) - ? bundleType.BundleType.SHAREDFLOW - : bundleType.BundleType.APIPROXY, - }, - externalPluginsDirectory: program.externalPluginsDirectory, - excluded: {}, - maxWarnings: -1, - profile: "apigee", -}; -if (!isNaN(program.maxWarnings)) { - configuration.maxWarnings = Number.parseInt(program.maxWarnings); -} + const configuration = { + debug: true, + source: { + type: sourceType, + path: resolvedPath, + sourcePath: sourcePath, + bundleType: resolvedPath.endsWith(bundleType.BundleType.SHAREDFLOW) + ? bundleType.BundleType.SHAREDFLOW + : bundleType.BundleType.APIPROXY, + }, + externalPluginsDirectory: program.externalPluginsDirectory, + excluded: {}, + maxWarnings: -1, + profile: "apigee", + }; + + if (!isNaN(program.maxWarnings)) { + configuration.maxWarnings = Number.parseInt(program.maxWarnings); + } -if (program.formatter) { - configuration.formatter = program.formatter || "json.js"; -} + if (program.formatter) { + configuration.formatter = program.formatter || "json.js"; + } -if (program.quiet) { - configuration.output = "none"; -} + if (program.quiet) { + configuration.output = "none"; + } -if (program.ignoreDirectives) { - configuration.ignoreDirectives = true; -} + if (program.ignoreDirectives) { + configuration.ignoreDirectives = true; + } -if (program.excluded && typeof program.excluded === "string") { - configuration.excluded = program.excluded - .split(",") - .map((s) => s.trim()) - .reduce((acc, item) => ((acc[item] = true), acc), {}); -} + if (program.excluded && typeof program.excluded === "string") { + configuration.excluded = program.excluded + .split(",") + .map((s) => s.trim()) + .reduce((acc, item) => ((acc[item] = true), acc), {}); + } -if (program.write) { - configuration.writePath = program.write; -} + if (program.write) { + configuration.writePath = program.write; + } -if (program.profile) { - configuration.profile = program.profile; -} + if (program.profile) { + configuration.profile = program.profile; + } -bl.lint(configuration); + bl.lint(configuration); +})(); diff --git a/lib/package/downloader.js b/lib/package/downloader.js new file mode 100644 index 0000000..11f0e04 --- /dev/null +++ b/lib/package/downloader.js @@ -0,0 +1,110 @@ +/* + Copyright © 2024 Google LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +const fs = require("fs"), + { Readable } = require("stream"), + { finished } = require("stream/promises"), + path = require("path"), + tmp = require("tmp"), + child_process = require("node:child_process"), + debug = require("debug")("apigeelint:download"); + +const downloadBundle = async (downloadSpec) => { + // 0. validate the input. it should be org:ORGNAME,api:APINAME or org:ORGNAME,sf:SHAREDFLOWNAME + let parts = downloadSpec.split(","); + let invalidArgument = () => { + console.log( + "Specify the value in the form org:ORGNAME,api:APINAME or org:ORGNAME,sf:SHAREDFLOWNAME", + ); + process.exit(1); + }; + if (!parts || (parts.length != 2 && parts.length != 3)) { + invalidArgument(); + } + let orgparts = parts[0].split(":"); + if (!orgparts || orgparts.length != 2 || orgparts[0] != "org") { + invalidArgument(); + } + let assetparts = parts[1].split(":"); + if (!assetparts || assetparts.length != 2) { + invalidArgument(); + } + if (assetparts[0] != "api" && assetparts[0] != "sf") { + invalidArgument(); + } + + let providedToken = null; + if (parts.length == 3) { + let tokenParts = parts[2].split(":"); + if (!tokenParts || tokenParts.length != 2 || tokenParts[0] != "token") { + invalidArgument(); + } + providedToken = tokenParts[1]; + } + + const execOptions = { + // cwd: proxyDir, // I think i do not care + encoding: "utf8", + }; + try { + // 1. use the provided token, or get a new one using gcloud. This may fail. + let accessToken = + providedToken || + child_process.execSync("gcloud auth print-access-token", execOptions); + // 2. inquire the revisions + let flavor = assetparts[0] == "api" ? "apis" : "sharedflows"; + const urlbase = `https://apigee.googleapis.com/v1/organizations/${orgparts[1]}/${flavor}`; + const headers = { + Accept: "application/json", + Authorization: `Bearer ${accessToken}`, + }; + + let url = `${urlbase}/${assetparts[1]}/revisions`; + let revisionsResponse = await fetch(url, { method: "GET", headers }); + + // 3. export the latest revision + if (!revisionsResponse.ok) { + throw new Error(`HTTP error: ${revisionsResponse.status}, on GET ${url}`); + } + const revisions = await revisionsResponse.json(); + revisions.sort(); + const rev = revisions[revisions.length - 1]; + url = `${urlbase}/${assetparts[1]}/revisions/${rev}?format=bundle`; + + const tmpdir = tmp.dirSync({ + prefix: `apigeelint-download-${assetparts[0]}`, + keep: false, + }); + const pathToDownloadedAsset = path.join( + tmpdir.name, + `${assetparts[1]}-rev${rev}.zip`, + ); + const stream = fs.createWriteStream(pathToDownloadedAsset); + const { body } = await fetch(url, { method: "GET", headers }); + await finished(Readable.fromWeb(body).pipe(stream)); + return pathToDownloadedAsset; + } catch (ex) { + // Possible causes: No gcloud cli found, or myriad other circumstances. + // Show the error message and first line of stack trace. + console.log(ex.stack.split("\n", 2).join("\n")); + console.log("cannot download the bundle from Apigee. Cannot continue."); + process.exit(1); + } +}; + +module.exports = { + downloadBundle, +}; From 98b05de54a1a5e022174e837db0e6e9c86efa280 Mon Sep 17 00:00:00 2001 From: DinoChiesa Date: Thu, 19 Dec 2024 06:46:12 +0000 Subject: [PATCH 4/4] remove dead code relating to the Edge management server --- cli.js | 4 - lib/package/Bundle.js | 148 ++++++--------------- lib/package/ManagementServer.js | 171 ------------------------ lib/package/processMgmtServer.js | 219 ------------------------------- 4 files changed, 40 insertions(+), 502 deletions(-) delete mode 100644 lib/package/ManagementServer.js delete mode 100644 lib/package/processMgmtServer.js diff --git a/cli.js b/cli.js index 72795e4..59cbd2b 100755 --- a/cli.js +++ b/cli.js @@ -95,10 +95,6 @@ const findBundle = (p) => { "-e, --excluded [value]", "The comma separated list of tests to exclude (default: none)", ) - // .option("-M, --mgmtserver [value]", "Apigee management server") - // .option("-u, --user [value]", "Apigee user account") - // .option("-p, --password [value]", "Apigee password") - // .option("-o, --organization [value]", "Apigee organization") .option( "-x, --externalPluginsDirectory [value]", "Relative or full path to an external plugins directory", diff --git a/lib/package/Bundle.js b/lib/package/Bundle.js index b1296f9..4c7049d 100644 --- a/lib/package/Bundle.js +++ b/lib/package/Bundle.js @@ -39,7 +39,7 @@ function _buildEndpoints(folder, tag, bundle, processFunction) { if (endptFile.endsWith(".xml") && fs.lstatSync(fqFname).isFile()) { const doc = xpath.select( tag, - new Dom().parseFromString(fs.readFileSync(fqFname).toString()) + new Dom().parseFromString(fs.readFileSync(fqFname).toString()), ); doc.forEach((element) => processFunction(element, fqFname, bundle)); } @@ -67,7 +67,12 @@ function buildsharedflows(bundle) { processFunction = function (element, fqFname, bundle) { bundle.proxyEndpoints = bundle.proxyEndpoints || []; bundle.proxyEndpoints.push( - new Endpoint(element, bundle, fqFname, bundleType.BundleType.SHAREDFLOW) + new Endpoint( + element, + bundle, + fqFname, + bundleType.BundleType.SHAREDFLOW, + ), ); }; _buildEndpoints(folder, tag, bundle, processFunction); @@ -92,10 +97,10 @@ function buildPolicies(bundle) { const ext = policyFile.split(".").pop(); if (ext === "xml") { bundle.policies.push( - new Policy(bundle.proxyRoot + "/policies", policyFile, bundle) + new Policy(bundle.proxyRoot + "/policies", policyFile, bundle), ); } - } + }, ); } } @@ -110,7 +115,7 @@ function _buildResources(parent, path, resources) { _buildResources(parent, path + "/" + policyFile, resources); } else if (!policyFile.endsWith("~")) { resources.push( - new Resource(parent, path + "/" + policyFile, policyFile) + new Resource(parent, path + "/" + policyFile, policyFile), ); } }); @@ -132,7 +137,7 @@ Bundle.prototype.getElement = function () { if (!this.element && fs.existsSync(filePath)) { debug(`getElement: filePath:${filePath}`); this.element = new Dom().parseFromString( - fs.readFileSync(filePath).toString() + fs.readFileSync(filePath).toString(), ); } return this.element; @@ -157,7 +162,7 @@ Bundle.prototype.getRevision = function () { try { const attr = xpath.select( `/${this.xPathName}/@revision`, - this.getElement() + this.getElement(), ); this.revision = (attr[0] && attr[0].value) || "undefined"; } catch (e) { @@ -192,7 +197,7 @@ function processFileSystem(config, bundle, cb) { .filter( (file) => fs.lstatSync(path.join(bundle.root, file)).isFile() && - file.endsWith(".xml") + file.endsWith(".xml"), ); if (files.length > 1) { @@ -211,7 +216,7 @@ function processFileSystem(config, bundle, cb) { warningCount: 0, fixableErrorCount: 0, fixableWarningCount: 0, - messages: [] + messages: [], }; //only run for proxy or shared flow root @@ -236,7 +241,7 @@ function processFileSystem(config, bundle, cb) { "No " + bundleTypeName + " folder found in: " + - JSON.stringify(folders) + JSON.stringify(folders), }); } } @@ -260,66 +265,9 @@ function Bundle(config, cb) { this.resolvedPath = config.source.path; this.sourceType = config.source.type; - if (config.source.type === "ManagementServer") { - //shared flow not implemented for management server - if (config.source.bundleType === bundleType.BundleType.SHAREDFLOW) { - throw "SharedFlows for management server not supported"; - } - - const ManagementServer = require("./ManagementServer.js"), - org = config.source.org, - api = config.source.api, - revision = config.source.revision, - tempFolder = "/tmp/" + api, - ms = new ManagementServer({ - org, - authorization: config.source.authorization - }); - - deleteFolderRecursive(tempFolder); - fs.mkdirSync(tempFolder); - const fileStream = fs.createWriteStream(tempFolder + "/apiproxy.zip"); - - ms.get( - "Bundle", - { org }, - { - api, - revision, - onRes: function (res) { - if (res.statusCode === 200) { - res.pipe(fileStream).on("close", function () { - decompress(tempFolder + "/apiproxy.zip", tempFolder).then( - (files) => { - //add the info to the config object here - config.source.path = tempFolder + "/apiproxy"; - processFileSystem(config, bundle, cb); - } - ); - }); - } else { - if (cb) { - cb(null, { - status: res.statusCode, - message: res.statusMessage - }); - } - } - } - }, - function (body) { - //callback on bundle downloaded - if (body.error) { - console.log(body.error); - //bundle.addMessage(JSON.stringify(body.error)); - deleteFolderRecursive(tempFolder); - } - } - ); - } else { - processFileSystem(config, bundle, cb); - } + processFileSystem(config, bundle, cb); } + const isFile = (spec) => { const exists = fs.existsSync(spec), stats = exists && fs.lstatSync(spec); @@ -353,7 +301,7 @@ Bundle.prototype.getReport = function (cb) { ) { try { const docElement = new Dom().parseFromString( - fs.readFileSync(item.filePath).toString() + fs.readFileSync(item.filePath).toString(), ).documentElement; const directives = lintUtil.findDirectives(docElement); d(JSON.stringify(directives, null, 2)); @@ -361,7 +309,7 @@ Bundle.prototype.getReport = function (cb) { const keep = (item) => { d("checking " + JSON.stringify(item)); const directivesForThisRule = directives.filter((directive) => - directive.disable.includes(item.ruleId) + directive.disable.includes(item.ruleId), ); /* * If there is a directive specifying the ruleId for this message @@ -373,21 +321,21 @@ Bundle.prototype.getReport = function (cb) { directivesForThisRule.length == 0 || (item.line && !directivesForThisRule.find( - (d) => d.line + 1 == item.line + (d) => d.line + 1 == item.line, )) || (!item.line && !directivesForThisRule.find( - (d) => d.line == 2 || d.line == 3 + (d) => d.line == 2 || d.line == 3, )) ); }; item.messages = item.messages.filter(keep); // update totals item.warningCount = item.messages.filter( - (m) => m.severity == 1 + (m) => m.severity == 1, ).length; item.errorCount = item.messages.filter( - (m) => m.severity == 2 + (m) => m.severity == 2, ).length; } } catch (e) { @@ -456,7 +404,7 @@ Bundle.prototype.onBundle = function (pluginFunction, cb) { Bundle.prototype.onPolicies = function (pluginFunction, cb) { this.getPolicies().forEach((policy) => - pluginFunction(policy, getcb(`policy '${policy.getName()}'`)) + pluginFunction(policy, getcb(`policy '${policy.getName()}'`)), ); cb(null, {}); }; @@ -472,8 +420,8 @@ Bundle.prototype.onSteps = function (pluginFunction, callback) { proxies.forEach((ep) => ep.onSteps( pluginFunction, - getcb(`STEP proxyendpoint '${ep.getName()}'`) - ) + getcb(`STEP proxyendpoint '${ep.getName()}'`), + ), ); } else { debug("no proxyEndpoints"); @@ -482,8 +430,8 @@ Bundle.prototype.onSteps = function (pluginFunction, callback) { targets.forEach((ep) => ep.onSteps( pluginFunction, - getcb(`STEP targetendpoint '${ep.getName()}'`) - ) + getcb(`STEP targetendpoint '${ep.getName()}'`), + ), ); } else { debug("no targetEndpoints"); @@ -505,16 +453,16 @@ Bundle.prototype.onConditions = function (pluginFunction, callback) { proxies.forEach((ep) => ep.onConditions( pluginFunction, - getcb(`COND proxyendpoint '${ep.getName()}'`) - ) + getcb(`COND proxyendpoint '${ep.getName()}'`), + ), ); } if (targets && targets.length > 0) { targets.forEach((ep) => ep.onConditions( pluginFunction, - getcb(`COND targetendpoint '${ep.getName()}'`) - ) + getcb(`COND targetendpoint '${ep.getName()}'`), + ), ); } } catch (exc1) { @@ -530,7 +478,7 @@ Bundle.prototype.onResources = function (pluginFunction, cb) { bundle .getResources() .forEach((re) => - pluginFunction(re, getcb(`resource '${re.getFileName()}'`)) + pluginFunction(re, getcb(`resource '${re.getFileName()}'`)), ); } cb(null, {}); @@ -539,7 +487,7 @@ Bundle.prototype.onResources = function (pluginFunction, cb) { Bundle.prototype.onFaultRules = function (pluginFunction, cb) { if (this.getFaultRules()) { this.getFaultRules().forEach( - (fr) => fr && pluginFunction(fr, getcb(`faultrule '${fr.getName()}'`)) + (fr) => fr && pluginFunction(fr, getcb(`faultrule '${fr.getName()}'`)), ); } cb(null, {}); @@ -548,7 +496,7 @@ Bundle.prototype.onFaultRules = function (pluginFunction, cb) { Bundle.prototype.onDefaultFaultRules = function (pluginFunction, cb) { if (this.getDefaultFaultRules()) { this.getDefaultFaultRules().forEach( - (dfr) => dfr && pluginFunction(dfr, getcb("defaultfaultrule")) + (dfr) => dfr && pluginFunction(dfr, getcb("defaultfaultrule")), ); } cb(null, {}); @@ -558,7 +506,7 @@ Bundle.prototype.onProxyEndpoints = function (pluginFunction, cb) { const eps = this.getProxyEndpoints(); if (eps && eps.length > 0) { eps.forEach((ep) => - pluginFunction(ep, getcb(`PEP proxyendpoint '${ep.getName()}'`)) + pluginFunction(ep, getcb(`PEP proxyendpoint '${ep.getName()}'`)), ); } cb(null, {}); @@ -568,7 +516,7 @@ Bundle.prototype.onTargetEndpoints = function (pluginFunction, cb) { const eps = this.getTargetEndpoints(); if (eps && eps.length > 0) { eps.forEach((ep) => - pluginFunction(ep, getcb(`TEP targetendpoint '${ep.getName()}'`)) + pluginFunction(ep, getcb(`TEP targetendpoint '${ep.getName()}'`)), ); } cb(null, {}); @@ -629,7 +577,7 @@ Bundle.prototype.getFaultRules = function () { if (!this.faultRules) { this.faultRules = this.getEndpoints().reduce( (a, ep) => [...a, ep.getFaultRules()], - [] + [], ); } return this.faultRules; @@ -639,7 +587,7 @@ Bundle.prototype.getDefaultFaultRules = function () { if (!this.defaultFaultRules) { this.defaultFaultRules = this.getEndpoints().reduce( (a, ep) => [...a, ep.getDefaultFaultRule()], - [] + [], ); } return this.defaultFaultRules; @@ -666,26 +614,10 @@ Bundle.prototype.summarize = function () { policies: this.getPolicies() ? this.getPolicies().map((po) => po.summarize()) - : [] + : [], }; return summary; }; -function deleteFolderRecursive(path) { - if (fs.existsSync(path)) { - fs.readdirSync(path).forEach(function (file) { - const curPath = path + "/" + file; - if (fs.lstatSync(curPath).isDirectory()) { - // recurse - deleteFolderRecursive(curPath); - } else { - // delete file - fs.unlinkSync(curPath); - } - }); - fs.rmdirSync(path); - } -} - //Public module.exports = Bundle; diff --git a/lib/package/ManagementServer.js b/lib/package/ManagementServer.js deleted file mode 100644 index 8c26f7a..0000000 --- a/lib/package/ManagementServer.js +++ /dev/null @@ -1,171 +0,0 @@ -/* - Copyright 2019 Google LLC - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -const uris = { - getEnviros: function(aConfig) { - return "/v1/organizations/" + aConfig.org + "/environments/"; - }, - getOrgAPIs: function(aConfig) { - return "/v1/organizations/" + aConfig.org + "/apis"; - }, - getRevisions: function(aConfig, args) { - return "/v1/organizations/" + aConfig.org + "/apis/" + args; - }, - getBundle: function(aConfig, args) { - return ( - "/v1/o/" + - aConfig.org + - "/apis/" + - args.api + - "/revisions/" + - args.revision + - "?format=bundle" - ); - } - }, - https = require("https"), - os = require("os"); - -ManagementServer.prototype.get = function(uri, aConfig, args, callback) { - aConfig = this.verifyConfig(aConfig); - this.mgmtServerQueue.push({ uri, aConfig, args, callback }); - if (!this.sending) { - makeNextCall(this); - } -}; - -function makeNextCall(ms) { - var call = ms.mgmtServerQueue.shift(); - if (call) { - ms.sending = true; - _get(ms, call.uri, call.aConfig, call.args, function(body, res) { - if (call.callback) { - call.callback(body, res); - makeNextCall(ms); - } - }); - } else { - ms.sending = false; - } -} - -function _get(ms, uri, aConfig, args, callback) { - var data = ""; - - if (aConfig.attempts > 1) { - console.log("processing " + aConfig.attempts + " attempt"); - } - - var options = { - host: aConfig.mgmtApiHost, - port: 443, - path: uris["get" + uri](aConfig, args), - method: "GET", - headers: { - Accept: "application/json", - Authorization: aConfig.authorization - } - }; - - var req = https.request(options, function(res) { - if (res.statusCode === 403) { - callback(data, res); - } else if (res.statusCode >= 300) { - ms.get(uri, aConfig, args, callback); - } - if (args.onRes) { - args.onRes(res, aConfig); - } else { - res.on("data", function(d) { - data += d; - }); - res.on("error", function(e) { - ms.get(uri, aConfig, args, callback); - }); - } - res.on("end", function() { - callback(data, res); - }); - }); - - req.on("error", function(e) { - ms.get(uri, aConfig, args, callback); - }); - req.end(); -} - -ManagementServer.prototype.verifyConfig = function(aConfig) { - aConfig.maxAttempts = aConfig.maxAttempts || this.setup.maxAttempts; - aConfig.attempts = aConfig.attempts || 1; - aConfig.authorization = aConfig.authorization || this.setup.authorization; - aConfig.retryDelay = aConfig.retryDelay || this.setup.retryDelay; - aConfig.mgmtApiHost = aConfig.mgmtApiHost || this.setup.mgmtApiHost; - return aConfig; -}; -ManagementServer.prototype.getAuthorization = function() { - return this.setup.authorization; -}; -ManagementServer.prototype.getHost = function() { - return this.setup.mgmtApiHost; -}; - -function ManagementServer(setup) { - if (typeof setup === "string") { - setup = { org: setup }; - } else { - setup = setup || {}; - } - - if (!setup.user && !setup.password) { - try { - var lintConfig = require(os.homedir() + "/.apigeeLintConfig.json"); - if (lintConfig[setup.org]) { - setup.user = lintConfig[setup.org].user; - setup.password = lintConfig[setup.org].password; - } else if (lintConfig._default) { - setup.user = lintConfig._default.user; - setup.password = lintConfig._default.password; - } - } catch (e) { - //no config init values otherwise - } - } - - this.setup = {}; - this.setup.maxAttempts = setup.maxAttempts || 5; - this.setup.timeout = setup.timeout || 180000; - this.setup.retryDelay = setup.retryDelay || 300; - this.setup.mgmtApiHost = setup.mgmtApiHost || "api.enterprise.apigee.com"; - this.setup.authorization = - setup.authorization || - (setup.user && - setup.password && - "Basic " + - new Buffer(setup.user + ":" + setup.password).toString("base64")) || - (process.env.au && - process.env.as && - "Basic " + - new Buffer(process.env.au + ":" + process.env.as).toString("base64")); - if (!this.setup.authorization) { - throw new Error( - "must provide authorization or .user and .secret or have environment variables au and as" - ); - } - this.mgmtServerQueue = []; - this.sending = false; -} - -module.exports = ManagementServer; diff --git a/lib/package/processMgmtServer.js b/lib/package/processMgmtServer.js deleted file mode 100644 index be5e965..0000000 --- a/lib/package/processMgmtServer.js +++ /dev/null @@ -1,219 +0,0 @@ -/* - Copyright 2019 Google LLC - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -//module to allow for interaction with mgmt server -const decompress = require("decompress"), - fs = require("fs"), - bl = require("./bundleLinter.js"), - ManagementServer = require("./ManagementServer.js"); - -var apis = [], - org = "davidwallen2014", - tempFolder = "./tmp/", - serial = 0, - rootDir = process.cwd(), - completed = {}; - -readCompletedFile(); - -deleteFolderRecursive(tempFolder); -fs.mkdirSync(tempFolder); -var ms = new ManagementServer(org); - -ms.get("OrgAPIs", { org }, [], function(body, res) { - function handleAPI(index) { - var revisions = []; - index = index || 0; - if (index < apis.length) { - var api = apis[index]; - - function handleRevision(revIndex) { - revIndex = revIndex || 0; - if (revIndex < revisions.length) { - var revision = revisions[revIndex]; - if (!isCompleted(ms, org, api, revision)) { - var proxyDir = - tempFolder + org + "-" + api + "-" + revision + "-" + serial++; - fs.mkdirSync(proxyDir); - var fileStream = fs.createWriteStream(proxyDir + "/apiproxy.zip"); - - ms.get( - "Bundle", - { org }, - { - api, - revision, - onRes: function(res, aConfig) { - if (res.statusCode === 200) { - res.pipe(fileStream).on("close", function() { - console.log("starting lint of " + proxyDir); - decompress( - proxyDir + "/apiproxy.zip", - proxyDir - ).then(files => { - bl.lint( - { - source: { - type: "filesystem", - path: proxyDir + "/apiproxy/" - }, - apiUpload: { - destPath: - "https://csdata-test.apigee.net/v1/lintresults", - authorization: ms.getAuthorization(), - organization: aConfig.org - }, - output: function(msg) { - fs.writeFile( - tempFolder + - "/" + - org + - "-" + - api + - "-" + - revision + - ".json", - msg, - function(err) { - msg = null; - if (err) { - return console.log(err); - } - } - ); - msg = null; - } - }, - function() { - addCompleted(ms, org, api, revision); - deleteFolderRecursive(proxyDir); - handleRevision(++revIndex); - } - ); - }); - }); - } - } - }, - function(body, res) { - //callback on bundle downloaded - if (body.error) { - console.log(body); - console.log("timeout bundle download: " + proxyDir); - console.log("cleaning up " + proxyDir); - - fs.writeFile( - tempFolder + - "/" + - org + - "-" + - api + - "-" + - revision + - "-error.json", - body, - function(err) { - if (err) { - return console.log(JSON.stringify(err)); - } - } - ); - console.log("abandonded lint: " + proxyDir); - - deleteFolderRecursive(proxyDir); - handleRevision(++revIndex); - } else { - console.log("completed bundle download: " + proxyDir); - } - } - ); - } else { - handleAPI(++index); - } - } - } - - ms.get("Revisions", { org }, api, function(body, res) { - //callback for recisions - if (body.error) { - console.log(body); - } else { - revisions = JSON.parse(body).revision; - handleRevision(); - } - }); - } - } - //call back once we have OrgAPIs - if (res.statusCode == 403 || res.statusCode == 401) { - throw new Error(res.statusMessage); - } - apis = JSON.parse(body); - handleAPI(); -}); - -function deleteFolderRecursive(path) { - if (fs.existsSync(path)) { - fs.readdirSync(path).forEach(function(file) { - var curPath = path + "/" + file; - if (fs.lstatSync(curPath).isDirectory()) { - // recurse - deleteFolderRecursive(curPath); - } else { - // delete file - fs.unlinkSync(curPath); - } - }); - fs.rmdirSync(path); - } -} - -function getCompletedKey(ms, org, api, revision) { - return ms.getHost() + "-" + org + "-" + api + "-" + revision; -} -function isCompleted(ms, org, api, revision) { - var result = completed[getCompletedKey(ms, org, api, revision)]; - return result || false; -} - -function addCompleted(ms, org, api, revision) { - readCompletedFile(); - completed[getCompletedKey(ms, org, api, revision)] = true; - writeCompletedFile(); -} - -function writeCompletedFile() { - fs.writeFile(rootDir + "/completed.json", JSON.stringify(completed), function( - err - ) { - if (err) { - return console.log(err); - } - }); - return completed; -} - -function readCompletedFile() { - try { - completed = JSON.parse( - fs.readFileSync(rootDir + "/completed.json", "utf8") - ); - } catch (e) { - console.log("initing completed file"); - completed = {}; - writeCompletedFile(); - } -}