diff --git a/cli.js b/cli.js index 72795e4..59cbd2b 100755 --- a/cli.js +++ b/cli.js @@ -95,10 +95,6 @@ const findBundle = (p) => { "-e, --excluded [value]", "The comma separated list of tests to exclude (default: none)", ) - // .option("-M, --mgmtserver [value]", "Apigee management server") - // .option("-u, --user [value]", "Apigee user account") - // .option("-p, --password [value]", "Apigee password") - // .option("-o, --organization [value]", "Apigee organization") .option( "-x, --externalPluginsDirectory [value]", "Relative or full path to an external plugins directory", diff --git a/lib/package/Bundle.js b/lib/package/Bundle.js index b1296f9..4c7049d 100644 --- a/lib/package/Bundle.js +++ b/lib/package/Bundle.js @@ -39,7 +39,7 @@ function _buildEndpoints(folder, tag, bundle, processFunction) { if (endptFile.endsWith(".xml") && fs.lstatSync(fqFname).isFile()) { const doc = xpath.select( tag, - new Dom().parseFromString(fs.readFileSync(fqFname).toString()) + new Dom().parseFromString(fs.readFileSync(fqFname).toString()), ); doc.forEach((element) => processFunction(element, fqFname, bundle)); } @@ -67,7 +67,12 @@ function buildsharedflows(bundle) { processFunction = function (element, fqFname, bundle) { bundle.proxyEndpoints = bundle.proxyEndpoints || []; bundle.proxyEndpoints.push( - new Endpoint(element, bundle, fqFname, bundleType.BundleType.SHAREDFLOW) + new Endpoint( + element, + bundle, + fqFname, + bundleType.BundleType.SHAREDFLOW, + ), ); }; _buildEndpoints(folder, tag, bundle, processFunction); @@ -92,10 +97,10 @@ function buildPolicies(bundle) { const ext = policyFile.split(".").pop(); if (ext === "xml") { bundle.policies.push( - new Policy(bundle.proxyRoot + "/policies", policyFile, bundle) + new Policy(bundle.proxyRoot + "/policies", policyFile, bundle), ); } - } + }, ); } } @@ -110,7 +115,7 @@ function _buildResources(parent, path, resources) { _buildResources(parent, path + "/" + policyFile, resources); } else if (!policyFile.endsWith("~")) { resources.push( - new Resource(parent, path + "/" + policyFile, policyFile) + new Resource(parent, path + "/" + policyFile, policyFile), ); } }); @@ -132,7 +137,7 @@ Bundle.prototype.getElement = function () { if (!this.element && fs.existsSync(filePath)) { debug(`getElement: filePath:${filePath}`); this.element = new Dom().parseFromString( - fs.readFileSync(filePath).toString() + fs.readFileSync(filePath).toString(), ); } return this.element; @@ -157,7 +162,7 @@ Bundle.prototype.getRevision = function () { try { const attr = xpath.select( `/${this.xPathName}/@revision`, - this.getElement() + this.getElement(), ); this.revision = (attr[0] && attr[0].value) || "undefined"; } catch (e) { @@ -192,7 +197,7 @@ function processFileSystem(config, bundle, cb) { .filter( (file) => fs.lstatSync(path.join(bundle.root, file)).isFile() && - file.endsWith(".xml") + file.endsWith(".xml"), ); if (files.length > 1) { @@ -211,7 +216,7 @@ function processFileSystem(config, bundle, cb) { warningCount: 0, fixableErrorCount: 0, fixableWarningCount: 0, - messages: [] + messages: [], }; //only run for proxy or shared flow root @@ -236,7 +241,7 @@ function processFileSystem(config, bundle, cb) { "No " + bundleTypeName + " folder found in: " + - JSON.stringify(folders) + JSON.stringify(folders), }); } } @@ -260,66 +265,9 @@ function Bundle(config, cb) { this.resolvedPath = config.source.path; this.sourceType = config.source.type; - if (config.source.type === "ManagementServer") { - //shared flow not implemented for management server - if (config.source.bundleType === bundleType.BundleType.SHAREDFLOW) { - throw "SharedFlows for management server not supported"; - } - - const ManagementServer = require("./ManagementServer.js"), - org = config.source.org, - api = config.source.api, - revision = config.source.revision, - tempFolder = "/tmp/" + api, - ms = new ManagementServer({ - org, - authorization: config.source.authorization - }); - - deleteFolderRecursive(tempFolder); - fs.mkdirSync(tempFolder); - const fileStream = fs.createWriteStream(tempFolder + "/apiproxy.zip"); - - ms.get( - "Bundle", - { org }, - { - api, - revision, - onRes: function (res) { - if (res.statusCode === 200) { - res.pipe(fileStream).on("close", function () { - decompress(tempFolder + "/apiproxy.zip", tempFolder).then( - (files) => { - //add the info to the config object here - config.source.path = tempFolder + "/apiproxy"; - processFileSystem(config, bundle, cb); - } - ); - }); - } else { - if (cb) { - cb(null, { - status: res.statusCode, - message: res.statusMessage - }); - } - } - } - }, - function (body) { - //callback on bundle downloaded - if (body.error) { - console.log(body.error); - //bundle.addMessage(JSON.stringify(body.error)); - deleteFolderRecursive(tempFolder); - } - } - ); - } else { - processFileSystem(config, bundle, cb); - } + processFileSystem(config, bundle, cb); } + const isFile = (spec) => { const exists = fs.existsSync(spec), stats = exists && fs.lstatSync(spec); @@ -353,7 +301,7 @@ Bundle.prototype.getReport = function (cb) { ) { try { const docElement = new Dom().parseFromString( - fs.readFileSync(item.filePath).toString() + fs.readFileSync(item.filePath).toString(), ).documentElement; const directives = lintUtil.findDirectives(docElement); d(JSON.stringify(directives, null, 2)); @@ -361,7 +309,7 @@ Bundle.prototype.getReport = function (cb) { const keep = (item) => { d("checking " + JSON.stringify(item)); const directivesForThisRule = directives.filter((directive) => - directive.disable.includes(item.ruleId) + directive.disable.includes(item.ruleId), ); /* * If there is a directive specifying the ruleId for this message @@ -373,21 +321,21 @@ Bundle.prototype.getReport = function (cb) { directivesForThisRule.length == 0 || (item.line && !directivesForThisRule.find( - (d) => d.line + 1 == item.line + (d) => d.line + 1 == item.line, )) || (!item.line && !directivesForThisRule.find( - (d) => d.line == 2 || d.line == 3 + (d) => d.line == 2 || d.line == 3, )) ); }; item.messages = item.messages.filter(keep); // update totals item.warningCount = item.messages.filter( - (m) => m.severity == 1 + (m) => m.severity == 1, ).length; item.errorCount = item.messages.filter( - (m) => m.severity == 2 + (m) => m.severity == 2, ).length; } } catch (e) { @@ -456,7 +404,7 @@ Bundle.prototype.onBundle = function (pluginFunction, cb) { Bundle.prototype.onPolicies = function (pluginFunction, cb) { this.getPolicies().forEach((policy) => - pluginFunction(policy, getcb(`policy '${policy.getName()}'`)) + pluginFunction(policy, getcb(`policy '${policy.getName()}'`)), ); cb(null, {}); }; @@ -472,8 +420,8 @@ Bundle.prototype.onSteps = function (pluginFunction, callback) { proxies.forEach((ep) => ep.onSteps( pluginFunction, - getcb(`STEP proxyendpoint '${ep.getName()}'`) - ) + getcb(`STEP proxyendpoint '${ep.getName()}'`), + ), ); } else { debug("no proxyEndpoints"); @@ -482,8 +430,8 @@ Bundle.prototype.onSteps = function (pluginFunction, callback) { targets.forEach((ep) => ep.onSteps( pluginFunction, - getcb(`STEP targetendpoint '${ep.getName()}'`) - ) + getcb(`STEP targetendpoint '${ep.getName()}'`), + ), ); } else { debug("no targetEndpoints"); @@ -505,16 +453,16 @@ Bundle.prototype.onConditions = function (pluginFunction, callback) { proxies.forEach((ep) => ep.onConditions( pluginFunction, - getcb(`COND proxyendpoint '${ep.getName()}'`) - ) + getcb(`COND proxyendpoint '${ep.getName()}'`), + ), ); } if (targets && targets.length > 0) { targets.forEach((ep) => ep.onConditions( pluginFunction, - getcb(`COND targetendpoint '${ep.getName()}'`) - ) + getcb(`COND targetendpoint '${ep.getName()}'`), + ), ); } } catch (exc1) { @@ -530,7 +478,7 @@ Bundle.prototype.onResources = function (pluginFunction, cb) { bundle .getResources() .forEach((re) => - pluginFunction(re, getcb(`resource '${re.getFileName()}'`)) + pluginFunction(re, getcb(`resource '${re.getFileName()}'`)), ); } cb(null, {}); @@ -539,7 +487,7 @@ Bundle.prototype.onResources = function (pluginFunction, cb) { Bundle.prototype.onFaultRules = function (pluginFunction, cb) { if (this.getFaultRules()) { this.getFaultRules().forEach( - (fr) => fr && pluginFunction(fr, getcb(`faultrule '${fr.getName()}'`)) + (fr) => fr && pluginFunction(fr, getcb(`faultrule '${fr.getName()}'`)), ); } cb(null, {}); @@ -548,7 +496,7 @@ Bundle.prototype.onFaultRules = function (pluginFunction, cb) { Bundle.prototype.onDefaultFaultRules = function (pluginFunction, cb) { if (this.getDefaultFaultRules()) { this.getDefaultFaultRules().forEach( - (dfr) => dfr && pluginFunction(dfr, getcb("defaultfaultrule")) + (dfr) => dfr && pluginFunction(dfr, getcb("defaultfaultrule")), ); } cb(null, {}); @@ -558,7 +506,7 @@ Bundle.prototype.onProxyEndpoints = function (pluginFunction, cb) { const eps = this.getProxyEndpoints(); if (eps && eps.length > 0) { eps.forEach((ep) => - pluginFunction(ep, getcb(`PEP proxyendpoint '${ep.getName()}'`)) + pluginFunction(ep, getcb(`PEP proxyendpoint '${ep.getName()}'`)), ); } cb(null, {}); @@ -568,7 +516,7 @@ Bundle.prototype.onTargetEndpoints = function (pluginFunction, cb) { const eps = this.getTargetEndpoints(); if (eps && eps.length > 0) { eps.forEach((ep) => - pluginFunction(ep, getcb(`TEP targetendpoint '${ep.getName()}'`)) + pluginFunction(ep, getcb(`TEP targetendpoint '${ep.getName()}'`)), ); } cb(null, {}); @@ -629,7 +577,7 @@ Bundle.prototype.getFaultRules = function () { if (!this.faultRules) { this.faultRules = this.getEndpoints().reduce( (a, ep) => [...a, ep.getFaultRules()], - [] + [], ); } return this.faultRules; @@ -639,7 +587,7 @@ Bundle.prototype.getDefaultFaultRules = function () { if (!this.defaultFaultRules) { this.defaultFaultRules = this.getEndpoints().reduce( (a, ep) => [...a, ep.getDefaultFaultRule()], - [] + [], ); } return this.defaultFaultRules; @@ -666,26 +614,10 @@ Bundle.prototype.summarize = function () { policies: this.getPolicies() ? this.getPolicies().map((po) => po.summarize()) - : [] + : [], }; return summary; }; -function deleteFolderRecursive(path) { - if (fs.existsSync(path)) { - fs.readdirSync(path).forEach(function (file) { - const curPath = path + "/" + file; - if (fs.lstatSync(curPath).isDirectory()) { - // recurse - deleteFolderRecursive(curPath); - } else { - // delete file - fs.unlinkSync(curPath); - } - }); - fs.rmdirSync(path); - } -} - //Public module.exports = Bundle; diff --git a/lib/package/ManagementServer.js b/lib/package/ManagementServer.js deleted file mode 100644 index 8c26f7a..0000000 --- a/lib/package/ManagementServer.js +++ /dev/null @@ -1,171 +0,0 @@ -/* - Copyright 2019 Google LLC - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -const uris = { - getEnviros: function(aConfig) { - return "/v1/organizations/" + aConfig.org + "/environments/"; - }, - getOrgAPIs: function(aConfig) { - return "/v1/organizations/" + aConfig.org + "/apis"; - }, - getRevisions: function(aConfig, args) { - return "/v1/organizations/" + aConfig.org + "/apis/" + args; - }, - getBundle: function(aConfig, args) { - return ( - "/v1/o/" + - aConfig.org + - "/apis/" + - args.api + - "/revisions/" + - args.revision + - "?format=bundle" - ); - } - }, - https = require("https"), - os = require("os"); - -ManagementServer.prototype.get = function(uri, aConfig, args, callback) { - aConfig = this.verifyConfig(aConfig); - this.mgmtServerQueue.push({ uri, aConfig, args, callback }); - if (!this.sending) { - makeNextCall(this); - } -}; - -function makeNextCall(ms) { - var call = ms.mgmtServerQueue.shift(); - if (call) { - ms.sending = true; - _get(ms, call.uri, call.aConfig, call.args, function(body, res) { - if (call.callback) { - call.callback(body, res); - makeNextCall(ms); - } - }); - } else { - ms.sending = false; - } -} - -function _get(ms, uri, aConfig, args, callback) { - var data = ""; - - if (aConfig.attempts > 1) { - console.log("processing " + aConfig.attempts + " attempt"); - } - - var options = { - host: aConfig.mgmtApiHost, - port: 443, - path: uris["get" + uri](aConfig, args), - method: "GET", - headers: { - Accept: "application/json", - Authorization: aConfig.authorization - } - }; - - var req = https.request(options, function(res) { - if (res.statusCode === 403) { - callback(data, res); - } else if (res.statusCode >= 300) { - ms.get(uri, aConfig, args, callback); - } - if (args.onRes) { - args.onRes(res, aConfig); - } else { - res.on("data", function(d) { - data += d; - }); - res.on("error", function(e) { - ms.get(uri, aConfig, args, callback); - }); - } - res.on("end", function() { - callback(data, res); - }); - }); - - req.on("error", function(e) { - ms.get(uri, aConfig, args, callback); - }); - req.end(); -} - -ManagementServer.prototype.verifyConfig = function(aConfig) { - aConfig.maxAttempts = aConfig.maxAttempts || this.setup.maxAttempts; - aConfig.attempts = aConfig.attempts || 1; - aConfig.authorization = aConfig.authorization || this.setup.authorization; - aConfig.retryDelay = aConfig.retryDelay || this.setup.retryDelay; - aConfig.mgmtApiHost = aConfig.mgmtApiHost || this.setup.mgmtApiHost; - return aConfig; -}; -ManagementServer.prototype.getAuthorization = function() { - return this.setup.authorization; -}; -ManagementServer.prototype.getHost = function() { - return this.setup.mgmtApiHost; -}; - -function ManagementServer(setup) { - if (typeof setup === "string") { - setup = { org: setup }; - } else { - setup = setup || {}; - } - - if (!setup.user && !setup.password) { - try { - var lintConfig = require(os.homedir() + "/.apigeeLintConfig.json"); - if (lintConfig[setup.org]) { - setup.user = lintConfig[setup.org].user; - setup.password = lintConfig[setup.org].password; - } else if (lintConfig._default) { - setup.user = lintConfig._default.user; - setup.password = lintConfig._default.password; - } - } catch (e) { - //no config init values otherwise - } - } - - this.setup = {}; - this.setup.maxAttempts = setup.maxAttempts || 5; - this.setup.timeout = setup.timeout || 180000; - this.setup.retryDelay = setup.retryDelay || 300; - this.setup.mgmtApiHost = setup.mgmtApiHost || "api.enterprise.apigee.com"; - this.setup.authorization = - setup.authorization || - (setup.user && - setup.password && - "Basic " + - new Buffer(setup.user + ":" + setup.password).toString("base64")) || - (process.env.au && - process.env.as && - "Basic " + - new Buffer(process.env.au + ":" + process.env.as).toString("base64")); - if (!this.setup.authorization) { - throw new Error( - "must provide authorization or .user and .secret or have environment variables au and as" - ); - } - this.mgmtServerQueue = []; - this.sending = false; -} - -module.exports = ManagementServer; diff --git a/lib/package/processMgmtServer.js b/lib/package/processMgmtServer.js deleted file mode 100644 index be5e965..0000000 --- a/lib/package/processMgmtServer.js +++ /dev/null @@ -1,219 +0,0 @@ -/* - Copyright 2019 Google LLC - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -//module to allow for interaction with mgmt server -const decompress = require("decompress"), - fs = require("fs"), - bl = require("./bundleLinter.js"), - ManagementServer = require("./ManagementServer.js"); - -var apis = [], - org = "davidwallen2014", - tempFolder = "./tmp/", - serial = 0, - rootDir = process.cwd(), - completed = {}; - -readCompletedFile(); - -deleteFolderRecursive(tempFolder); -fs.mkdirSync(tempFolder); -var ms = new ManagementServer(org); - -ms.get("OrgAPIs", { org }, [], function(body, res) { - function handleAPI(index) { - var revisions = []; - index = index || 0; - if (index < apis.length) { - var api = apis[index]; - - function handleRevision(revIndex) { - revIndex = revIndex || 0; - if (revIndex < revisions.length) { - var revision = revisions[revIndex]; - if (!isCompleted(ms, org, api, revision)) { - var proxyDir = - tempFolder + org + "-" + api + "-" + revision + "-" + serial++; - fs.mkdirSync(proxyDir); - var fileStream = fs.createWriteStream(proxyDir + "/apiproxy.zip"); - - ms.get( - "Bundle", - { org }, - { - api, - revision, - onRes: function(res, aConfig) { - if (res.statusCode === 200) { - res.pipe(fileStream).on("close", function() { - console.log("starting lint of " + proxyDir); - decompress( - proxyDir + "/apiproxy.zip", - proxyDir - ).then(files => { - bl.lint( - { - source: { - type: "filesystem", - path: proxyDir + "/apiproxy/" - }, - apiUpload: { - destPath: - "https://csdata-test.apigee.net/v1/lintresults", - authorization: ms.getAuthorization(), - organization: aConfig.org - }, - output: function(msg) { - fs.writeFile( - tempFolder + - "/" + - org + - "-" + - api + - "-" + - revision + - ".json", - msg, - function(err) { - msg = null; - if (err) { - return console.log(err); - } - } - ); - msg = null; - } - }, - function() { - addCompleted(ms, org, api, revision); - deleteFolderRecursive(proxyDir); - handleRevision(++revIndex); - } - ); - }); - }); - } - } - }, - function(body, res) { - //callback on bundle downloaded - if (body.error) { - console.log(body); - console.log("timeout bundle download: " + proxyDir); - console.log("cleaning up " + proxyDir); - - fs.writeFile( - tempFolder + - "/" + - org + - "-" + - api + - "-" + - revision + - "-error.json", - body, - function(err) { - if (err) { - return console.log(JSON.stringify(err)); - } - } - ); - console.log("abandonded lint: " + proxyDir); - - deleteFolderRecursive(proxyDir); - handleRevision(++revIndex); - } else { - console.log("completed bundle download: " + proxyDir); - } - } - ); - } else { - handleAPI(++index); - } - } - } - - ms.get("Revisions", { org }, api, function(body, res) { - //callback for recisions - if (body.error) { - console.log(body); - } else { - revisions = JSON.parse(body).revision; - handleRevision(); - } - }); - } - } - //call back once we have OrgAPIs - if (res.statusCode == 403 || res.statusCode == 401) { - throw new Error(res.statusMessage); - } - apis = JSON.parse(body); - handleAPI(); -}); - -function deleteFolderRecursive(path) { - if (fs.existsSync(path)) { - fs.readdirSync(path).forEach(function(file) { - var curPath = path + "/" + file; - if (fs.lstatSync(curPath).isDirectory()) { - // recurse - deleteFolderRecursive(curPath); - } else { - // delete file - fs.unlinkSync(curPath); - } - }); - fs.rmdirSync(path); - } -} - -function getCompletedKey(ms, org, api, revision) { - return ms.getHost() + "-" + org + "-" + api + "-" + revision; -} -function isCompleted(ms, org, api, revision) { - var result = completed[getCompletedKey(ms, org, api, revision)]; - return result || false; -} - -function addCompleted(ms, org, api, revision) { - readCompletedFile(); - completed[getCompletedKey(ms, org, api, revision)] = true; - writeCompletedFile(); -} - -function writeCompletedFile() { - fs.writeFile(rootDir + "/completed.json", JSON.stringify(completed), function( - err - ) { - if (err) { - return console.log(err); - } - }); - return completed; -} - -function readCompletedFile() { - try { - completed = JSON.parse( - fs.readFileSync(rootDir + "/completed.json", "utf8") - ); - } catch (e) { - console.log("initing completed file"); - completed = {}; - writeCompletedFile(); - } -}