diff --git a/README.md b/README.md index 7e5a1f3..3dba457 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,11 @@ **Lambdium allows you to run a Selenium Webdriver script written in Javascript inside of an AWS Lambda function bundled with [Headless Chromium](https://developers.google.com/web/updates/2017/04/headless-chrome).** -You can use this AWS Lambda function to: +You can use this AWS Lambda function by itself or with other AWS services to: * Run many concurrent selenium scripts at the same time without worrying about the infrastructure -* Configure Cloudwatch events to run script(s) on a schedule ([example app](/examples/apps/scheduled-event.yaml)) +* Run execute a selenium script via an HTTP call using API Gateway +* Configure Cloudwatch events to run a script on a schedule ([example app](/examples/apps/scheduled-event.yaml)) * Integrate selenium tests running in Chrome into different event-driven workflows (like CodeDeploy checks, webhooks, or uploads to an S3 bucket) Since this Lambda function is written using node.js, you can run almost any script written for [selenium-webdriver](https://www.npmjs.com/package/selenium-webdriver). Example scripts can be found in the `examples` directory. diff --git a/event-api.json b/event-api.json new file mode 100644 index 0000000..da0b2eb --- /dev/null +++ b/event-api.json @@ -0,0 +1,59 @@ +{ + "body": "LS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS03NjgyM2JlYzc1MjY0MGQ0DQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9InNjcmlwdCI7IGZpbGVuYW1lPSJ2aXNpdGdvb2dsZS5qcyINCkNvbnRlbnQtVHlwZTogYXBwbGljYXRpb24vb2N0ZXQtc3RyZWFtDQoNCi8vIFNhbXBsZSBzZWxlbmltdW0td2ViZHJpdmVyIHNjcmlwdCB0aGF0IHZpc2l0cyBnb29nbGUuY29tCi8vIFRoaXMgdXNlcyB0aGUgc2VsZW5pdW0td2ViZHJpdmVyIDMuNCBwYWNrYWdlLgovLyBEb2NzOiBodHRwczovL3NlbGVuaXVtaHEuZ2l0aHViLmlvL3NlbGVuaXVtL2RvY3MvYXBpL2phdmFzY3JpcHQvbW9kdWxlL3NlbGVuaXVtLXdlYmRyaXZlci9pbmRleC5odG1sCi8vICRicm93c2VyID0gd2ViZHJpdmVyIHNlc3Npb24KLy8gJGRyaXZlciA9IGRyaXZlciBsaWJyYXJpZXMKLy8gY29uc29sZS5sb2cgd2lsbCBvdXRwdXQgdG8gQVdTIExhbWJkYSBsb2dzICh2aWEgQ2xvdWR3YXRjaCkKCmNvbnNvbGUubG9nKCdBYm91dCB0byB2aXNpdCBnb29nbGUuY29tLi4uJyk7CiRicm93c2VyLmdldCgnaHR0cDovL3d3dy5nb29nbGUuY29tL25jcicpOwokYnJvd3Nlci5maW5kRWxlbWVudCgkZHJpdmVyLkJ5Lm5hbWUoJ2J0bksnKSkuY2xpY2soKTsKJGJyb3dzZXIud2FpdCgkZHJpdmVyLnVudGlsLnRpdGxlSXMoJ0dvb2dsZScpLCAxMDAwKTsKJGJyb3dzZXIuZ2V0VGl0bGUoKS50aGVuKGZ1bmN0aW9uKHRpdGxlKSB7CiAgICBjb25zb2xlLmxvZygidGl0bGUgaXM6ICIgKyB0aXRsZSk7CiAgICBjb25zb2xlLmxvZygnRmluaXNoZWQgcnVubmluZyBzY3JpcHQhJyk7Cn0pOwoNCi0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tNzY4MjNiZWM3NTI2NDBkNC0tDQo=", + "resource": "/{proxy+}", + "requestContext": { + "resourceId": "123456", + "apiId": "1234567890", + "resourcePath": "/{proxy+}", + "httpMethod": "POST", + "requestId": "c6af9ac6-7b61-11e6-9a41-93e8deadbeef", + "accountId": "123456789012", + "identity": { + "apiKey": null, + "userArn": null, + "cognitoAuthenticationType": null, + "caller": null, + "userAgent": "Custom User Agent String", + "user": null, + "cognitoIdentityPoolId": null, + "cognitoIdentityId": null, + "cognitoAuthenticationProvider": null, + "sourceIp": "127.0.0.1", + "accountId": null + }, + "stage": "prod" + }, + "queryStringParameters": { + "foo": "bar" + }, + "headers": { + "Via": "1.1 08f323deadbeefa7af34d5feb414ce27.cloudfront.net (CloudFront)", + "Accept-Language": "en-US,en;q=0.8", + "CloudFront-Is-Desktop-Viewer": "true", + "CloudFront-Is-SmartTV-Viewer": "false", + "CloudFront-Is-Mobile-Viewer": "false", + "X-Forwarded-For": "127.0.0.1, 127.0.0.2", + "CloudFront-Viewer-Country": "US", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Upgrade-Insecure-Requests": "1", + "X-Forwarded-Port": "443", + "Host": "1234567890.execute-api.us-east-1.amazonaws.com", + "X-Forwarded-Proto": "https", + "X-Amz-Cf-Id": "aaaaaaaaaae3VYQb9jd-nvCd-de396Uhbp027Y2JvkCPNLmGJHqlaA==", + "CloudFront-Is-Tablet-Viewer": "false", + "Cache-Control": "max-age=0", + "Content-Type": "multipart/form-data; boundary=------------------------76823bec752640d4", + "User-Agent": "Custom User Agent String", + "CloudFront-Forwarded-Proto": "https", + "Accept-Encoding": "gzip, deflate, sdch" + }, + "pathParameters": { + "proxy": "/examplepath" + }, + "httpMethod": "POST", + "stageVariables": { + "baz": "qux" + }, + "isBase64Encoded": true, + "path": "/examplepath" +} \ No newline at end of file diff --git a/examples/apps/api-gateway.yaml b/examples/apps/api-gateway.yaml new file mode 100644 index 0000000..4b3530d --- /dev/null +++ b/examples/apps/api-gateway.yaml @@ -0,0 +1,46 @@ +AWSTemplateFormatVersion : '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: selenium with headless chromium +Resources: + Lambdium: + Type: AWS::Serverless::Function + Properties: + Handler: index.postApiGatewayHandler + Runtime: nodejs6.10 + FunctionName: lambdium + Description: headless chromium running selenium + # This needs to be fairly large: chromium needs a lot of memory + MemorySize: 1156 + Timeout: 20 + Environment: + Variables: + CLEAR_TMP: "true" + # packaged lambdium archive @ v0.2 + CodeUri: <> + Events: + RunScript: + Properties: + Method: POST + Path: '/runScript' + RestApiId: !Ref Api + Type: Api + Api: + Type: AWS::Serverless::Api + Properties: + Name: RunScriptAPI + StageName: Prod + DefinitionBody: + swagger: "2.0" + schemes: + - "https" + paths: + '/runScript': + post: + responses: {} + x-amazon-apigateway-integration: + uri: !Sub arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${Lambdium.Arn}/invocations + passthroughBehavior: "when_no_match" + httpMethod: "POST" + type: aws_proxy + x-amazon-apigateway-binary-media-types: + - "*/*" \ No newline at end of file diff --git a/index.js b/index.js index 08d26c1..a697f31 100644 --- a/index.js +++ b/index.js @@ -1,10 +1,13 @@ -const webdriver = require('selenium-webdriver'); -const child = require('child_process'); -const fs = require('fs'); const chromium = require('./lib/chromium'); const sandbox = require('./lib/sandbox'); const log = require('lambda-log'); +const apiHandler = require('./lib/api-handler'); + +if (process.env.DEBUG_ENV || process.env.SAM_LOCAL) { + log.config.debug = true; + log.config.dev = true; +} log.info('Loading function'); @@ -13,53 +16,21 @@ if (!process.env.CLEAN_SESSIONS) { $browser = chromium.createSession(); } -const parseScriptInput = (event) => { - const inputParam = event.Base64Script || process.env.BASE64_SCRIPT; - if (typeof inputParam !== 'string') { - return null - } - - return Buffer.from(inputParam, 'base64').toString('utf8'); -} - +// Handler for POST events from API gateway +// curl -v -F "script=@examples/visitgoogle.js" <> +exports.postApiGatewayHandler = apiHandler; + +// Default function event handler +// Accepts events: +// * {"Base64Script": "<>"} +// * {"pageUrl": "<>"} +// Accepts environment variables: +// * BASE64_SCRIPT: encoded selenium script +// * PAGE_URL: URI to visit exports.handler = (event, context, callback) => { - context.callbackWaitsForEmptyEventLoop = false; - - if (process.env.CLEAN_SESSIONS) { - log.info('attempting to clear /tmp directory') - log.info(child.execSync('rm -rf /tmp/core*').toString()); - } - - if (process.env.DEBUG_ENV || process.env.SAM_LOCAL) { - log.config.debug = true; - log.config.dev = true; - } - - if (process.env.LOG_DEBUG) { - log.debug(child.execSync('pwd').toString()); - log.debug(child.execSync('ls -lhtra .').toString()); - log.debug(child.execSync('ls -lhtra /tmp').toString()); - } - - log.info(`Received event: ${JSON.stringify(event, null, 2)}`); - - // Creates a new session on each event (instead of reusing for performance benefits) - if (process.env.CLEAN_SESSIONS) { - $browser = chromium.createSession(); - } - - var opts = { - browser: $browser, - driver: webdriver - }; + $browser = sandbox.initBrowser(event, context); - // Determine script to run: either a 1) base64-encoded selenium script or 2) a URL to visit - var inputBuffer = parseScriptInput(event); - if (inputBuffer !== null) { - opts.scriptText = inputBuffer; - } else if (event.pageUrl || process.env.PAGE_URL) { - opts.pageUrl = event.pageUrl || process.env.PAGE_URL; - } + var opts = sandbox.buildOptions(event, $browser); sandbox.executeScript(opts, function(err) { if (process.env.LOG_DEBUG) { diff --git a/lib/api-handler.js b/lib/api-handler.js new file mode 100644 index 0000000..bbaa9ab --- /dev/null +++ b/lib/api-handler.js @@ -0,0 +1,67 @@ +const busboy = require('busboy'); +const path = require('path'); +const os = require('os'); + +const sandbox = require('./sandbox'); + +module.exports = function(event, context, callback) { + $browser = sandbox.initBrowser(event, context); + var errorMessage = ''; + const response = { + statusCode: 200, + headers: { + "Content-Type": 'application/text', + "X-Error": errorMessage || null + }, + body: '', + isBase64Encoded: false + }; + var body = event.body; + if (event.isBase64Encoded) { + body = Buffer.from(event.body, 'base64').toString('utf8'); + } + var scriptFile = new Buffer(0) + + + const SCRIPT_FIELDNAME = 'script'; + + var contentType = event.headers['Content-Type'] || event.headers['content-type']; + var bb = new busboy({ headers: { 'content-type': contentType }}); + var result = {}; + bb.on('file', function (fieldname, file, filename, encoding, mimetype) { + file.on('data', data => { + result.file = data; + }); + + file.on('end', () => { + result.filename = filename; + result.contentType = mimetype; + }); + }) + .on('finish', () => { + + // Execute uploaded script + var scriptText = result.file.toString(); + var opts = sandbox.buildOptions(event, $browser); + opts.scriptText = scriptText; + + sandbox.executeScript(opts, function(err, output) { + if (err) { + response.headers['X-Error'] = err; + response.body = err; + response.statusCode = 500; + return callback(null, response); + } + response.body = output; + callback(null, response); + }); + }) + .on('error', err => { + response.headers['X-Error'] = err; + response.body = err; + response.statusCode = 500; + callback(null, response); + }); + + bb.end(body); + }; \ No newline at end of file diff --git a/lib/sandbox.js b/lib/sandbox.js index 376cad7..09293bf 100644 --- a/lib/sandbox.js +++ b/lib/sandbox.js @@ -1,9 +1,47 @@ const vm = require('vm'); const log = require('lambda-log'); +const chromium = require('./chromium'); +const webdriver = require('selenium-webdriver'); + +exports.initBrowser = function(event, context) { + context.callbackWaitsForEmptyEventLoop = false; + + if (process.env.CLEAN_SESSIONS) { + log.info('attempting to clear /tmp directory') + log.info(child.execSync('rm -rf /tmp/core*').toString()); + } + + log.info(`Received event: ${JSON.stringify(event, null, 2)}`); + + // Creates a new session on each event (instead of reusing for performance benefits) + if (process.env.CLEAN_SESSIONS) { + $browser = chromium.createSession(); + } + return $browser; +}; + +exports.buildOptions = (event, browser) => { + var opts = opts = { + browser: $browser, + driver: webdriver + }; + + const inputParam = event.Base64Script || process.env.BASE64_SCRIPT; + if (typeof inputParam !== 'string') { + opts.pageUrl = event.pageUrl || process.env.PAGE_URL; + return opts; + } + + var inputBuffer = Buffer.from(inputParam, 'base64').toString('utf8'); + opts.scriptText = inputBuffer; + + return opts; +}; exports.executeScript = function(opts = {}, cb) { const browser = opts.browser; const driver = opts.driver; + var output = ''; var scriptText = opts.scriptText; // Just visit a web page if a script isn't specified @@ -20,6 +58,7 @@ exports.executeScript = function(opts = {}, cb) { log: function(){ var args = Array.prototype.slice.call(arguments); args.unshift('[lambdium-selenium]'); + output = `${output}\n${args.join(' ')}`; console.log.apply(console, args); } }; @@ -46,15 +85,15 @@ exports.executeScript = function(opts = {}, cb) { // Reuse existing session, likely some edge cases around this... if (process.env.CLEAN_SESSIONS) { browser.quit().then(function() { - cb(null); + cb(null, output); }); } else { browser.manage().deleteAllCookies().then(function() { return browser.get('about:blank').then(function() { - cb(null); + cb(null, output); }); }).catch(function(err) { - cb(err); + cb(err, output); }); } } diff --git a/package-lock.json b/package-lock.json index 6e67c8e..b542a73 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "lambdium", - "version": "0.1.0", + "version": "0.1.2", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -18,6 +18,33 @@ "concat-map": "0.0.1" } }, + "busboy": { + "version": "0.2.14", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-0.2.14.tgz", + "integrity": "sha1-bCpiLvz0fFe7vh4qnDetNseSVFM=", + "requires": { + "dicer": "0.2.5", + "readable-stream": "1.1.14" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "0.0.1", + "string_decoder": "0.10.31" + } + } + } + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -33,6 +60,33 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, + "dicer": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/dicer/-/dicer-0.2.5.tgz", + "integrity": "sha1-WZbAhrszIYyBLAkL3cCc0S+stw8=", + "requires": { + "readable-stream": "1.1.14", + "streamsearch": "0.1.2" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "0.0.1", + "string_decoder": "0.10.31" + } + } + } + }, "es6-promise": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-3.0.2.tgz", @@ -183,6 +237,11 @@ "xml2js": "0.4.17" } }, + "streamsearch": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz", + "integrity": "sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=" + }, "string_decoder": { "version": "0.10.31", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", diff --git a/package.json b/package.json index 4f535b5..884b26f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "lambdium", - "version": "0.1.2", + "version": "0.2.0", "description": "headless chromium in lambda prototype", "main": "index.js", "scripts": { @@ -9,6 +9,7 @@ "author": "Clay Smith", "license": "ISC", "dependencies": { + "busboy": "^0.2.14", "lambda-log": "^1.3.0", "selenium-webdriver": "^3.6.0" }