Merge branch 'develop' of repo.ch.internal:oss/spindle into develop

bbcarchdev · Aug 11, 2017 · aa980a5 · aa980a5
2 parents 12ec52d + 4cf9ec9
commit aa980a5
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 0 deletions.
diff --git a/quilt/module.c b/quilt/module.c
@@ -28,6 +28,7 @@ AWSS3BUCKET *spindle_bucket;
 char *spindle_cachepath;
 SQL *spindle_db;
 int spindle_s3_verbose;
+long spindle_s3_fetch_limit;
 
 static int spindle_cache_init_(void);
 static int spindle_cache_init_s3_(const char *bucket);
@@ -161,6 +162,10 @@ spindle_cache_init_s3_(const char *bucket)
 		aws_s3_set_secret(spindle_bucket, t);
 		free(t);
 	}
+
+	// As its in terms of kbs
+	spindle_s3_fetch_limit = 1024 * quilt_config_get_int("s3:fetch_limit", DEFAULT_SPINDLE_FETCH_LIMIT);
+
 	spindle_s3_verbose = quilt_config_get_bool("s3:verbose", 0);
 	return 0;
 }

diff --git a/quilt/p_spindle.h b/quilt/p_spindle.h
@@ -39,6 +39,8 @@
 
 # define SPINDLE_THRESHOLD              40
 
+# define DEFAULT_SPINDLE_FETCH_LIMIT	( 2 * 1024 )
+
 # define MIME_NQUADS                    "application/n-quads"
 
 /* Namespaces */

diff --git a/quilt/s3.c b/quilt/s3.c
@@ -24,6 +24,8 @@
 
 #include "p_spindle.h"
 
+extern long spindle_s3_fetch_limit;
+
 struct data_struct
 {
 	char *buf;
@@ -132,6 +134,13 @@ spindle_s3_write_(char *ptr, size_t size, size_t nemb, void *userdata)
 	data = (struct data_struct *) userdata;
 
 	size *= nemb;
+
+	if(spindle_s3_fetch_limit && size > spindle_s3_fetch_limit)
+	{
+		quilt_logf(LOG_WARNING, QUILT_PLUGIN_NAME ": S3: failing write due to size exceeding fetch limit. input_size:%u > fetch_limit:%u \n", size, spindle_s3_fetch_limit);
+		return 0;
+	}
+
 	if(data->pos + size >= data->size)
 	{
 		p = (char *) realloc(data->buf, data->size + size + 1);