Implement a single-bucket design.

benchmarks/100.webapps/110.dynamic-html/input.py

@@ -8,7 +8,7 @@
 def buckets_count():
     return (0, 0)
 
-def generate_input(data_dir, size, input_buckets, output_buckets, upload_func):
+def generate_input(data_dir, size, benchmarks_bucket, input_paths, output_paths, upload_func):
     input_config = {'username': 'testname'} 
     input_config['random_len'] = size_generators[size]
     return input_config

benchmarks/100.webapps/120.uploader/input.py

@@ -11,8 +11,9 @@
 def buckets_count():
     return (0, 1)
 
-def generate_input(data_dir, size, input_buckets, output_buckets, upload_func):
+def generate_input(data_dir, size, benchmarks_bucket, input_buckets, output_buckets, upload_func):
     input_config = {'object': {}, 'bucket': {}}
     input_config['object']['url'] = url_generators[size]
+    input_config['bucket']['bucket'] = benchmarks_bucket
     input_config['bucket']['output'] = output_buckets[0]
     return input_config

benchmarks/100.webapps/120.uploader/nodejs/function.js

@@ -15,7 +15,8 @@ function streamToPromise(stream) {
 }
 
 exports.handler = async function(event) {
-  let output_bucket = event.bucket.output
+  let bucket = event.bucket.bucket
+  let output_prefix = event.bucket.output
   let url = event.object.url
   let upload_key = path.basename(url)
   let download_path = path.join('/tmp', upload_key)
@@ -26,10 +27,10 @@ exports.handler = async function(event) {
   var keyName;
   let upload = promise.then(
     async () => {
-      [keyName, promise] = storage_handler.upload(output_bucket, upload_key, download_path);
+      [keyName, promise] = storage_handler.upload(bucket, path.join(output_prefix, upload_key), download_path);
       await promise;
     }
   );
   await upload;
-  return {bucket: output_bucket, url: url, key: keyName}
+  return {bucket: bucket, url: url, key: keyName}
 };

benchmarks/100.webapps/120.uploader/python/function.py

@@ -11,7 +11,8 @@
 
 def handler(event):
 
-    output_bucket = event.get('bucket').get('output')
+    bucket = event.get('bucket').get('bucket')
+    output_prefix = event.get('bucket').get('output')
     url = event.get('object').get('url')
     name = os.path.basename(url)
     download_path = '/tmp/{}'.format(name)
@@ -22,14 +23,14 @@ def handler(event):
     process_end = datetime.datetime.now()
 
     upload_begin = datetime.datetime.now()
-    key_name = client.upload(output_bucket, name, download_path)
+    key_name = client.upload(bucket, os.path.join(output_prefix, name), download_path)
     upload_end = datetime.datetime.now()
 
     process_time = (process_end - process_begin) / datetime.timedelta(microseconds=1)
     upload_time = (upload_end - upload_begin) / datetime.timedelta(microseconds=1)
     return {
             'result': {
-                'bucket': output_bucket,
+                'bucket': bucket,
                 'url': url,
                 'key': key_name
             },

benchmarks/200.multimedia/210.thumbnailer/input.py

@@ -12,15 +12,18 @@ def buckets_count():
     :param output_buckets:
     :param upload_func: upload function taking three params(bucket_idx, key, filepath)
 '''
-def generate_input(data_dir, size, input_buckets, output_buckets, upload_func):
+def generate_input(data_dir, size, benchmarks_bucket, input_paths, output_paths, upload_func):
+
     for file in glob.glob(os.path.join(data_dir, '*.jpg')):
         img = os.path.relpath(file, data_dir)
         upload_func(0, img, file)
+
     #TODO: multiple datasets
     input_config = {'object': {}, 'bucket': {}}
     input_config['object']['key'] = img
     input_config['object']['width'] = 200
     input_config['object']['height'] = 200
-    input_config['bucket']['input'] = input_buckets[0]
-    input_config['bucket']['output'] = output_buckets[0]
+    input_config['bucket']['bucket'] = benchmarks_bucket
+    input_config['bucket']['input'] = input_paths[0]
+    input_config['bucket']['output'] = output_paths[0]
     return input_config

benchmarks/200.multimedia/210.thumbnailer/nodejs/function.js

@@ -5,22 +5,24 @@ const sharp = require('sharp'),
 let storage_handler = new storage.storage();
 
 exports.handler = async function(event) {
-  input_bucket = event.bucket.input
-  output_bucket = event.bucket.output
+
+  bucket = event.bucket.bucket
+  input_prefix = event.bucket.input
+  output_prefix = event.bucket.output
   let key = event.object.key
   width = event.object.width
   height = event.object.height
   let pos = key.lastIndexOf('.');
   let upload_key = key.substr(0, pos < 0 ? key.length : pos) + '.png';
 
   const sharp_resizer = sharp().resize(width, height).png();
-  let read_promise = storage_handler.downloadStream(input_bucket, key);
-  let [writeStream, promise, uploadName] = storage_handler.uploadStream(output_bucket, upload_key);
+  let read_promise = storage_handler.downloadStream(bucket, path.join(input_prefix, key));
+  let [writeStream, promise, uploadName] = storage_handler.uploadStream(bucket, path.join(output_prefix, upload_key));
   read_promise.then(
     (input_stream) => {
       input_stream.pipe(sharp_resizer).pipe(writeStream);
     }
   );
   await promise;
-  return {bucket: output_bucket, key: uploadName}
+  return {bucket: output_prefix, key: uploadName}
 };

benchmarks/200.multimedia/210.thumbnailer/python/function.py

@@ -27,8 +27,9 @@ def resize_image(image_bytes, w, h):
 
 def handler(event):
 
-    input_bucket = event.get('bucket').get('input')
-    output_bucket = event.get('bucket').get('output')
+    bucket = event.get('bucket').get('bucket')
+    input_prefix = event.get('bucket').get('input')
+    output_prefix = event.get('bucket').get('output')
     key = unquote_plus(event.get('object').get('key'))
     width = event.get('object').get('width')
     height = event.get('object').get('height')
@@ -39,7 +40,7 @@ def handler(event):
     #resize_image(download_path, upload_path, width, height)
     #client.upload(output_bucket, key, upload_path)
     download_begin = datetime.datetime.now()
-    img = client.download_stream(input_bucket, key)
+    img = client.download_stream(bucket, os.path.join(input_prefix, key))
     download_end = datetime.datetime.now()
 
     process_begin = datetime.datetime.now()
@@ -48,15 +49,15 @@ def handler(event):
     process_end = datetime.datetime.now()
 
     upload_begin = datetime.datetime.now()
-    key_name = client.upload_stream(output_bucket, key, resized)
+    key_name = client.upload_stream(bucket, os.path.join(output_prefix, key), resized)
     upload_end = datetime.datetime.now()
 
     download_time = (download_end - download_begin) / datetime.timedelta(microseconds=1)
     upload_time = (upload_end - upload_begin) / datetime.timedelta(microseconds=1)
     process_time = (process_end - process_begin) / datetime.timedelta(microseconds=1)
     return {
             'result': {
-                'bucket': output_bucket,
+                'bucket': bucket,
                 'key': key_name
             },
             'measurement': {

benchmarks/200.multimedia/220.video-processing/input.py

@@ -12,7 +12,7 @@ def buckets_count():
     :param output_buckets:
     :param upload_func: upload function taking three params(bucket_idx, key, filepath)
 '''
-def generate_input(data_dir, size, input_buckets, output_buckets, upload_func):
+def generate_input(data_dir, size, benchmarks_bucket, input_paths, output_paths, upload_func):
     for file in glob.glob(os.path.join(data_dir, '*.mp4')):
         img = os.path.relpath(file, data_dir)
         upload_func(0, img, file)
@@ -21,6 +21,7 @@ def generate_input(data_dir, size, input_buckets, output_buckets, upload_func):
     input_config['object']['key'] = img
     input_config['object']['op'] = 'watermark'
     input_config['object']['duration'] = 1
-    input_config['bucket']['input'] = input_buckets[0]
-    input_config['bucket']['output'] = output_buckets[0]
+    input_config['bucket']['bucket'] = benchmarks_bucket
+    input_config['bucket']['input'] = input_paths[0]
+    input_config['bucket']['output'] = output_paths[0]
     return input_config

benchmarks/200.multimedia/220.video-processing/python/function.py

@@ -52,8 +52,10 @@ def transcode_mp3(video, duration, event):
 operations = { 'transcode' : transcode_mp3, 'extract-gif' : to_gif, 'watermark' : watermark }
 
 def handler(event):
-    input_bucket = event.get('bucket').get('input')
-    output_bucket = event.get('bucket').get('output')
+
+    bucket = event.get('bucket').get('bucket')
+    input_prefix = event.get('bucket').get('input')
+    output_prefix = event.get('bucket').get('output')
     key = event.get('object').get('key')
     duration = event.get('object').get('duration')
     op = event.get('object').get('op')
@@ -69,7 +71,7 @@ def handler(event):
         pass
 
     download_begin = datetime.datetime.now()
-    client.download(input_bucket, key, download_path)
+    client.download(bucket, os.path.join(input_prefix, key), download_path)
     download_size = os.path.getsize(download_path)
     download_stop = datetime.datetime.now()
 
@@ -80,16 +82,16 @@ def handler(event):
     upload_begin = datetime.datetime.now()
     filename = os.path.basename(upload_path)
     upload_size = os.path.getsize(upload_path)
-    client.upload(output_bucket, filename, upload_path)
+    upload_key = client.upload(bucket, os.path.join(output_prefix, filename), upload_path)
     upload_stop = datetime.datetime.now()
 
     download_time = (download_stop - download_begin) / datetime.timedelta(microseconds=1)
     upload_time = (upload_stop - upload_begin) / datetime.timedelta(microseconds=1)
     process_time = (process_end - process_begin) / datetime.timedelta(microseconds=1)
     return {
             'result': {
-                'bucket': output_bucket,
-                'key': filename
+                'bucket': bucket,
+                'key': upload_key
             },
             'measurement': {
                 'download_time': download_time,

benchmarks/300.utilities/311.compression/input.py

@@ -22,7 +22,7 @@ def upload_files(data_root, data_dir, upload_func):
     :param output_buckets:
     :param upload_func: upload function taking three params(bucket_idx, key, filepath)
 '''
-def generate_input(data_dir, size, input_buckets, output_buckets, upload_func):
+def generate_input(data_dir, size, benchmarks_bucket, input_paths, output_paths, upload_func):
 
     # upload different datasets
     datasets = []
@@ -32,6 +32,7 @@ def generate_input(data_dir, size, input_buckets, output_buckets, upload_func):
 
     input_config = {'object': {}, 'bucket': {}}
     input_config['object']['key'] = datasets[0]
-    input_config['bucket']['input'] = input_buckets[0]
-    input_config['bucket']['output'] = output_buckets[0]
+    input_config['bucket']['bucket'] = benchmarks_bucket
+    input_config['bucket']['input'] = input_paths[0]
+    input_config['bucket']['output'] = output_paths[0]
     return input_config

benchmarks/300.utilities/311.compression/python/function.py

@@ -18,14 +18,15 @@ def parse_directory(directory):
 
 def handler(event):
 
-    input_bucket = event.get('bucket').get('input')
-    output_bucket = event.get('bucket').get('output')
+    bucket = event.get('bucket').get('bucket')
+    input_prefix = event.get('bucket').get('input')
+    output_prefix = event.get('bucket').get('output')
     key = event.get('object').get('key')
     download_path = '/tmp/{}-{}'.format(key, uuid.uuid4())
     os.makedirs(download_path)
 
     s3_download_begin = datetime.datetime.now()
-    client.download_directory(input_bucket, key, download_path)
+    client.download_directory(bucket, os.path.join(input_prefix, key), download_path)
     s3_download_stop = datetime.datetime.now()
     size = parse_directory(download_path)
 
@@ -36,15 +37,15 @@ def handler(event):
     s3_upload_begin = datetime.datetime.now()
     archive_name = '{}.zip'.format(key)
     archive_size = os.path.getsize(os.path.join(download_path, archive_name))
-    key_name = client.upload(output_bucket, archive_name, os.path.join(download_path, archive_name))
+    key_name = client.upload(bucket, os.path.join(output_prefix, archive_name), os.path.join(download_path, archive_name))
     s3_upload_stop = datetime.datetime.now()
 
     download_time = (s3_download_stop - s3_download_begin) / datetime.timedelta(microseconds=1)
     upload_time = (s3_upload_stop - s3_upload_begin) / datetime.timedelta(microseconds=1)
     process_time = (compress_end - compress_begin) / datetime.timedelta(microseconds=1)
     return {
             'result': {
-                'bucket': output_bucket,
+                'bucket': bucket,
                 'key': key_name
             },
             'measurement': {

benchmarks/400.inference/411.image-recognition/input.py

@@ -21,7 +21,7 @@ def upload_files(data_root, data_dir, upload_func):
     :param output_buckets:
     :param upload_func: upload function taking three params(bucket_idx, key, filepath)
 '''
-def generate_input(data_dir, size, input_buckets, output_buckets, upload_func):
+def generate_input(data_dir, size, benchmarks_bucket, input_paths, output_paths, upload_func):
 
     # upload model
     model_name = 'resnet50-19c8e357.pth'
@@ -38,6 +38,7 @@ def generate_input(data_dir, size, input_buckets, output_buckets, upload_func):
     input_config = {'object': {}, 'bucket': {}}
     input_config['object']['model'] = model_name
     input_config['object']['input'] = input_images[0][0]
-    input_config['bucket']['model'] = input_buckets[0]
-    input_config['bucket']['input'] = input_buckets[1]
+    input_config['bucket']['bucket'] = benchmarks_bucket
+    input_config['bucket']['input'] = input_paths[1]
+    input_config['bucket']['model'] = input_paths[0]
     return input_config

benchmarks/400.inference/411.image-recognition/python/function.py

@@ -25,22 +25,23 @@
 
 def handler(event):
 
-    model_bucket = event.get('bucket').get('model')
-    input_bucket = event.get('bucket').get('input')
+    bucket = event.get('bucket').get('bucket')
+    input_prefix = event.get('bucket').get('input')
+    model_prefix = event.get('bucket').get('model')
     key = event.get('object').get('input')
     model_key = event.get('object').get('model')
     download_path = '/tmp/{}-{}'.format(key, uuid.uuid4())
 
     image_download_begin = datetime.datetime.now()
     image_path = download_path
-    client.download(input_bucket, key, download_path)
+    client.download(bucket, os.path.join(input_prefix, key), download_path)
     image_download_end = datetime.datetime.now()
 
     global model
     if not model:
         model_download_begin = datetime.datetime.now()
         model_path = os.path.join('/tmp', model_key)
-        client.download(model_bucket, model_key, model_path)
+        client.download(bucket, os.path.join(model_prefix, model_key), model_path)
         model_download_end = datetime.datetime.now()
         model_process_begin = datetime.datetime.now()
         model = resnet50(pretrained=False)

benchmarks/500.scientific/501.graph-pagerank/input.py

@@ -5,7 +5,7 @@
 }
 
 def buckets_count():
-    return (1, 1)
+    return (0, 0)
 
-def generate_input(data_dir, size, input_buckets, output_buckets, upload_func):
+def generate_input(data_dir, size, benchmarks_bucket, input_paths, output_paths, upload_func):
     return { 'size': size_generators[size] }

benchmarks/500.scientific/502.graph-mst/input.py

@@ -5,7 +5,7 @@
 }
 
 def buckets_count():
-    return (1, 1)
+    return (0, 0)
 
-def generate_input(data_dir, size, input_buckets, output_buckets, upload_func):
+def generate_input(data_dir, size, benchmarks_bucket, input_paths, output_paths, upload_func):
     return { 'size': size_generators[size] }

benchmarks/500.scientific/503.graph-bfs/input.py

@@ -5,7 +5,7 @@
 }
 
 def buckets_count():
-    return (1, 1)
+    return (0, 0)
 
-def generate_input(data_dir, size, input_buckets, output_buckets, upload_func):
+def generate_input(data_dir, size, benchmarks_bucket, input_paths, output_paths, upload_func):
     return { 'size': size_generators[size] }

benchmarks/500.scientific/504.dna-visualisation/input.py

@@ -3,12 +3,14 @@
 def buckets_count():
     return (1, 1)
 
-def generate_input(data_dir, size, input_buckets, output_buckets, upload_func):
+def generate_input(data_dir, size, benchmarks_bucket, input_paths, output_paths, upload_func):
+
     for file in glob.glob(os.path.join(data_dir, '*.fasta')):
         data = os.path.relpath(file, data_dir)
         upload_func(0, data, file)
     input_config = {'object': {}, 'bucket': {}}
     input_config['object']['key'] = data
-    input_config['bucket']['input'] = input_buckets[0]
-    input_config['bucket']['output'] = output_buckets[0]
+    input_config['bucket']['bucket'] = benchmarks_bucket
+    input_config['bucket']['input'] = input_paths[0]
+    input_config['bucket']['output'] = output_paths[0]
     return input_config