Skip to content

Commit

Permalink
post-kinesis.js: add handling for partial failures
Browse files Browse the repository at this point in the history
  • Loading branch information
jerith666 committed Aug 25, 2021
1 parent cd16102 commit df9bec6
Showing 1 changed file with 16 additions and 1 deletion.
17 changes: 16 additions & 1 deletion lib/post-kinesis.js
Original file line number Diff line number Diff line change
Expand Up @@ -68,5 +68,20 @@ exports.create = function(target, options) {
// This function sends messages to Amazon Kinesis
exports.send = function(service, target, records, callback) {
var entries = records.map(function(record) { return { PartitionKey: record.key, Data: record.data }; })
service.putRecords({ StreamName: target.destination, Records: entries }, callback);
service.putRecords({ StreamName: target.destination, Records: entries }, function(err, data){
/*A kinesis putRecords() request can sometimes partially succeed and partially fail. In order to prevent data loss,
we report this as a top-level failure of the send() operation, so that the lambda's retry & error reporting
processes can kick in and handle it.*/
if(err) {
callback(err,null);
} else {
if(data.FailedRecordCount) {
var totalRecords = data.Records && data.Records.length ? " (of " + data.Records.length + " total)"
: "";
callback(new Error(data.FailedRecordCount + totalRecords + " records failed in putRecords()"),null);
} else {
callback(null,data);
}
}
});
};

0 comments on commit df9bec6

Please sign in to comment.