From 3f901e123b2a9009713ef5eb1ae5ce949cd0620c Mon Sep 17 00:00:00 2001 From: Pierre Tomasina Date: Mon, 17 Jul 2017 00:50:11 +0200 Subject: [PATCH] Makeup --- LICENSE.txt | 21 +++++++++++++++++++++ README.md | 43 +++++++++++++++++++++++++++++++++++++++---- package.json | 30 ++++++++++++++++++++++++++---- 3 files changed, 86 insertions(+), 8 deletions(-) create mode 100644 LICENSE.txt diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..96308e3 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Pierre Tomasina + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 9f1af8a..0e8f5aa 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,43 @@ DynamoDB to Kinesis -This library let you batch a full DynamoDB table to dispatch JSON event to Kinesis Stream. +This library will scan a DynamoDB table to dispatch JSON event to Kinesis Stream. ## Benefits -1. Extract full data of table into a Kinesis Stream let you the possiblity to manage data as you want with a buffer of 24hours. +Extract full data of the table into a Kinesis Stream let you the possibility to manage data as you want with a buffer of 24hours. ## UseCases -Create a full new ElasticSearchIndex before enable DynamoDB Stream to ES +Create a complete new ElasticSearch index before enabling DynamoDB Stream to ES. + +## Installation + +``` +npm i -S dyki +``` + +## Usage + +The system is really simple, you just have to Instance a new client with `DyKi.Client` +and specify the DynamoDB table, Kinesis stream name, and the region associate. + +A 4th parameter is a configuration object: + - `delay: number` adjust the delay in millisecond between two scan + - `dyCapacityUnitLimit: number` defined a dynamodb capacity unit limit to stop scan if it's reached + - `progressCallbackInterval: number` interval in millisecond for calling progress callback + +then you can call the `run` method to start the scan stream to kinesis. +it returns a promise where your callback will call at the end of a full scan. +(until scan result do not contain lastEvaluatedKey). + +The run method can accept 3 optional arguments: + - the number of results paginates for each scan + - the ExclusiveStartKey object for dynamoDB (useful to start at a specific point of time) + - callback function calls in an interval to give information with the following object in argument: + - `lastEvaluatedKey: ` item primary key from the last scan + - `total: number` Total of items already analyzed + - `consumedUnitCapacity: number` The consumed unit capacity of table from the last scan -## Demo ```javascript 'use strict' @@ -49,3 +76,11 @@ client.run(25, startKey, progressCallback).then(info => { }); ``` +## Question, Contribution? + +If you have any question, recommendation, or improvement. +Or even if you use this class and it's useful for you, +thanks for taking a few minutes to open an issue and share your experience. + +Thanks. + diff --git a/package.json b/package.json index 8169758..00677e9 100644 --- a/package.json +++ b/package.json @@ -1,13 +1,35 @@ { "name": "dyki", - "version": "0.1.0", - "description": "Extract DynamoDB items to dispatch into Kinesis Stream", + "version": "1.0.0", + "description": "Scan DynamoDB items to dispatch into Kinesis Stream", + "keywords": [ + "aws", + "serverless", + "amazon web service", + "dynamodb", + "kinesis", + "stream", + "elastic" + ], + "repository": { + "type": "git", + "url": "https://github.com/pierozi/dyki" + }, + "bugs": "https://github.com/pierozi/dyki/issues", + "maintainers": [{ + "name": "Pierre Tomasina", + "web": "http://continuous.lu" + }], + "contributors": [{ + "name": "Pierre Tomasina", + "web": "http://continuous.lu" + }], "main": "src/main.js", "scripts": { "test": "node tests/scratch.js" }, - "author": "Pierre Tomasina (continuousphp.com)", - "license": "ISC", + "author": "Pierre Tomasina (continuous.lu)", + "license": "MIT", "dependencies": { "aws-sdk": "^2.84.0", "base-64": "^0.1.0",