diff --git a/.mypy.ini b/.mypy.ini index fece12c6..0c41d4cb 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -24,11 +24,11 @@ ignore_missing_imports = True [mypy-minio] ignore_missing_imports = True -[mypy-google.cloud] -ignore_missing_imports = True - -[mypy-google.api_core] +[mypy-google.*] ignore_missing_imports = True [mypy-testtools] ignore_missing_imports = True + +[mypy-redis] +ignore_missing_imports = True \ No newline at end of file diff --git a/benchmarks-data b/benchmarks-data index 6a17a460..9166d3f8 160000 --- a/benchmarks-data +++ b/benchmarks-data @@ -1 +1 @@ -Subproject commit 6a17a460f289e166abb47ea6298fb939e80e8beb +Subproject commit 9166d3f89621ad01919c8dd47bacdf04e36b890d diff --git a/benchmarks/600.workflows/610.gen/config.json b/benchmarks/600.workflows/610.gen/config.json new file mode 100644 index 00000000..8eae0824 --- /dev/null +++ b/benchmarks/600.workflows/610.gen/config.json @@ -0,0 +1,5 @@ +{ + "timeout": 120, + "memory": 128, + "languages": ["python"] +} diff --git a/benchmarks/600.workflows/610.gen/definition.json b/benchmarks/600.workflows/610.gen/definition.json new file mode 100644 index 00000000..d788231d --- /dev/null +++ b/benchmarks/600.workflows/610.gen/definition.json @@ -0,0 +1,48 @@ +{ + "root": "get_astros", + "states": { + "get_astros": { + "type": "task", + "func_name": "get_astros", + "next": "select_astros_number" + }, + "select_astros_number": { + "type": "switch", + "cases": [ + { + "var": "astros.number", + "op": "<", + "val": 10, + "next": "few_people" + }, + { + "var": "astros.number", + "op": ">=", + "val": 10, + "next": "many_people" + } + ], + "default": "few_people" + }, + "few_people": { + "type": "task", + "func_name": "few_people", + "next": "map_astros" + }, + "many_people": { + "type": "task", + "func_name": "many_people", + "next": "map_astros" + }, + "map_astros": { + "type": "map", + "array": "astros.people", + "func_name": "map_astros", + "next": "process_astros" + }, + "process_astros": { + "type": "task", + "func_name": "process_astros" + } + } +} \ No newline at end of file diff --git a/benchmarks/600.workflows/610.gen/input.py b/benchmarks/600.workflows/610.gen/input.py new file mode 100644 index 00000000..68f82e81 --- /dev/null +++ b/benchmarks/600.workflows/610.gen/input.py @@ -0,0 +1,5 @@ +def buckets_count(): + return (0, 0) + +def generate_input(data_dir, size, input_buckets, output_buckets, upload_func): + return dict() \ No newline at end of file diff --git a/benchmarks/600.workflows/610.gen/python/few_people.py b/benchmarks/600.workflows/610.gen/python/few_people.py new file mode 100644 index 00000000..9c70d9fb --- /dev/null +++ b/benchmarks/600.workflows/610.gen/python/few_people.py @@ -0,0 +1,5 @@ +def handler(event): + return { + "many_astros": False, + **event + } \ No newline at end of file diff --git a/benchmarks/600.workflows/610.gen/python/get_astros.py b/benchmarks/600.workflows/610.gen/python/get_astros.py new file mode 100644 index 00000000..627c6523 --- /dev/null +++ b/benchmarks/600.workflows/610.gen/python/get_astros.py @@ -0,0 +1,8 @@ +import requests + +def handler(event): + res = requests.get("http://api.open-notify.org/astros.json") + + return { + "astros": res.json() + } \ No newline at end of file diff --git a/benchmarks/600.workflows/610.gen/python/many_people.py b/benchmarks/600.workflows/610.gen/python/many_people.py new file mode 100644 index 00000000..2d339f32 --- /dev/null +++ b/benchmarks/600.workflows/610.gen/python/many_people.py @@ -0,0 +1,5 @@ +def handler(event): + return { + "many_astros": True, + **event + } \ No newline at end of file diff --git a/benchmarks/600.workflows/610.gen/python/map_astros.py b/benchmarks/600.workflows/610.gen/python/map_astros.py new file mode 100644 index 00000000..b98b5e9d --- /dev/null +++ b/benchmarks/600.workflows/610.gen/python/map_astros.py @@ -0,0 +1,7 @@ +def handler(elem): + name = elem["name"] + fn, ln = name.split(" ") + name = " ".join([ln, fn]) + elem["name_rev"] = name + + return elem \ No newline at end of file diff --git a/benchmarks/600.workflows/610.gen/python/process_astros.py b/benchmarks/600.workflows/610.gen/python/process_astros.py new file mode 100644 index 00000000..a981660e --- /dev/null +++ b/benchmarks/600.workflows/610.gen/python/process_astros.py @@ -0,0 +1,5 @@ +def handler(arr): + return { + "astros": arr, + "done": True + } \ No newline at end of file diff --git a/benchmarks/600.workflows/620.func_invo/config.json b/benchmarks/600.workflows/620.func_invo/config.json new file mode 100644 index 00000000..8eae0824 --- /dev/null +++ b/benchmarks/600.workflows/620.func_invo/config.json @@ -0,0 +1,5 @@ +{ + "timeout": 120, + "memory": 128, + "languages": ["python"] +} diff --git a/benchmarks/600.workflows/620.func_invo/definition.json b/benchmarks/600.workflows/620.func_invo/definition.json new file mode 100644 index 00000000..bd4ef736 --- /dev/null +++ b/benchmarks/600.workflows/620.func_invo/definition.json @@ -0,0 +1,10 @@ +{ + "root": "process", + "states": { + "process": { + "type": "loop", + "func_name": "process", + "count": 10 + } + } +} \ No newline at end of file diff --git a/benchmarks/600.workflows/620.func_invo/input.py b/benchmarks/600.workflows/620.func_invo/input.py new file mode 100644 index 00000000..237cc3f7 --- /dev/null +++ b/benchmarks/600.workflows/620.func_invo/input.py @@ -0,0 +1,11 @@ +size_generators = { + 'test' : 10, + 'small' : 2**10, + 'large': 2**15 +} + +def buckets_count(): + return (0, 0) + +def generate_input(data_dir, size, input_buckets, output_buckets, upload_func): + return { 'size': size_generators[size] } \ No newline at end of file diff --git a/benchmarks/600.workflows/620.func_invo/python/process.py b/benchmarks/600.workflows/620.func_invo/python/process.py new file mode 100644 index 00000000..807a9fb2 --- /dev/null +++ b/benchmarks/600.workflows/620.func_invo/python/process.py @@ -0,0 +1,14 @@ +from random import shuffle + +def handler(event): + size = int(event["size"]) if isinstance(event, dict) else len(event) + elems = list(range(size)) + shuffle(elems) + + data = "" + for i in elems: + data += str(i % 255) + if len(data) > size: + break + + return data[:size] \ No newline at end of file diff --git a/benchmarks/600.workflows/630.parallel/config.json b/benchmarks/600.workflows/630.parallel/config.json new file mode 100644 index 00000000..8eae0824 --- /dev/null +++ b/benchmarks/600.workflows/630.parallel/config.json @@ -0,0 +1,5 @@ +{ + "timeout": 120, + "memory": 128, + "languages": ["python"] +} diff --git a/benchmarks/600.workflows/630.parallel/definition.json b/benchmarks/600.workflows/630.parallel/definition.json new file mode 100644 index 00000000..cf5664b9 --- /dev/null +++ b/benchmarks/600.workflows/630.parallel/definition.json @@ -0,0 +1,15 @@ +{ + "root": "generate", + "states": { + "generate": { + "type": "task", + "func_name": "generate", + "next": "process" + }, + "process": { + "type": "map", + "func_name": "process", + "array": "buffer" + } + } +} \ No newline at end of file diff --git a/benchmarks/600.workflows/630.parallel/input.py b/benchmarks/600.workflows/630.parallel/input.py new file mode 100644 index 00000000..f80fafd1 --- /dev/null +++ b/benchmarks/600.workflows/630.parallel/input.py @@ -0,0 +1,11 @@ +size_generators = { + 'test' : 5, + 'small' : 100, + 'large': 1000 +} + +def buckets_count(): + return (0, 0) + +def generate_input(data_dir, size, input_buckets, output_buckets, upload_func): + return { 'size': size_generators[size] } \ No newline at end of file diff --git a/benchmarks/600.workflows/630.parallel/python/generate.py b/benchmarks/600.workflows/630.parallel/python/generate.py new file mode 100644 index 00000000..79a27dda --- /dev/null +++ b/benchmarks/600.workflows/630.parallel/python/generate.py @@ -0,0 +1,7 @@ +def handler(event): + size = int(event["size"]) + buffer = size * ["asdf"] + + return { + "buffer": buffer + } \ No newline at end of file diff --git a/benchmarks/600.workflows/630.parallel/python/process.py b/benchmarks/600.workflows/630.parallel/python/process.py new file mode 100644 index 00000000..41712b35 --- /dev/null +++ b/benchmarks/600.workflows/630.parallel/python/process.py @@ -0,0 +1,2 @@ +def handler(elem): + return elem[::-1] \ No newline at end of file diff --git a/benchmarks/wrappers/aws/python/handler.py b/benchmarks/wrappers/aws/python/handler_function.py similarity index 100% rename from benchmarks/wrappers/aws/python/handler.py rename to benchmarks/wrappers/aws/python/handler_function.py diff --git a/benchmarks/wrappers/aws/python/handler_workflow.py b/benchmarks/wrappers/aws/python/handler_workflow.py new file mode 100644 index 00000000..f95d01ff --- /dev/null +++ b/benchmarks/wrappers/aws/python/handler_workflow.py @@ -0,0 +1,56 @@ + +import datetime +import io +import json +import os +import sys +import uuid +import importlib + +# Add current directory to allow location of packages +sys.path.append(os.path.join(os.path.dirname(__file__), '.python_packages/lib/site-packages')) + +from redis import Redis + +def probe_cold_start(): + is_cold = False + fname = os.path.join("/tmp", "cold_run") + if not os.path.exists(fname): + is_cold = True + container_id = str(uuid.uuid4())[0:8] + with open(fname, "a") as f: + f.write(container_id) + else: + with open(fname, "r") as f: + container_id = f.read() + + return is_cold, container_id + + +def handler(event, context): + start = datetime.datetime.now().timestamp() + + workflow_name, func_name = context.function_name.split("___") + function = importlib.import_module(f"function.{func_name}") + res = function.handler(event) + + end = datetime.datetime.now().timestamp() + + is_cold, container_id = probe_cold_start() + payload = json.dumps({ + "func": func_name, + "start": start, + "end": end, + "is_cold": is_cold, + "container_id": container_id + }) + + redis = Redis(host={{REDIS_HOST}}, + port=6379, + decode_responses=True, + socket_connect_timeout=10) + + key = os.path.join(workflow_name, func_name, str(uuid.uuid4())[0:8]) + redis.set(key, payload) + + return res diff --git a/benchmarks/wrappers/aws/python/storage.py b/benchmarks/wrappers/aws/python/storage.py index f979d07c..c5cc1e7f 100644 --- a/benchmarks/wrappers/aws/python/storage.py +++ b/benchmarks/wrappers/aws/python/storage.py @@ -15,17 +15,17 @@ def __init__(self): @staticmethod def unique_name(name): name, extension = os.path.splitext(name) - return '{name}.{random}.{extension}'.format( + return '{name}.{random}{extension}'.format( name=name, extension=extension, random=str(uuid.uuid4()).split('-')[0] ) - + def upload(self, bucket, file, filepath): key_name = storage.unique_name(file) self.client.upload_file(filepath, bucket, key_name) return key_name - + def download(self, bucket, file, filepath): self.client.download_file(bucket, file, filepath) @@ -46,7 +46,7 @@ def download_stream(self, bucket, file): data = io.BytesIO() self.client.download_fileobj(bucket, file, data) return data.getbuffer() - + def get_instance(): if storage.instance is None: storage.instance = storage() diff --git a/benchmarks/wrappers/azure/python/fsm.py b/benchmarks/wrappers/azure/python/fsm.py new file mode 120000 index 00000000..b3891312 --- /dev/null +++ b/benchmarks/wrappers/azure/python/fsm.py @@ -0,0 +1 @@ +../../../../../serverless-benchmarks/sebs/faas/fsm.py \ No newline at end of file diff --git a/benchmarks/wrappers/azure/python/handler.py b/benchmarks/wrappers/azure/python/handler_function.py similarity index 96% rename from benchmarks/wrappers/azure/python/handler.py rename to benchmarks/wrappers/azure/python/handler_function.py index 5f7f14f2..ca17294d 100644 --- a/benchmarks/wrappers/azure/python/handler.py +++ b/benchmarks/wrappers/azure/python/handler_function.py @@ -6,7 +6,7 @@ # TODO: usual trigger # implement support for blob and others -def main(req: func.HttpRequest, context: func.Context) -> func.HttpResponse: +def main(req: func.HttpRequest, starter: str, context: func.Context) -> func.HttpResponse: income_timestamp = datetime.datetime.now().timestamp() req_json = req.get_json() if 'connection_string' in req_json: diff --git a/benchmarks/wrappers/azure/python/handler_workflow.py b/benchmarks/wrappers/azure/python/handler_workflow.py new file mode 100644 index 00000000..e54c6a89 --- /dev/null +++ b/benchmarks/wrappers/azure/python/handler_workflow.py @@ -0,0 +1,59 @@ +import datetime +import json +import os +import uuid +import importlib + +from azure.storage.blob import BlobServiceClient +import azure.functions as func +from redis import Redis + +def probe_cold_start(): + is_cold = False + fname = os.path.join("/tmp", "cold_run") + if not os.path.exists(fname): + is_cold = True + container_id = str(uuid.uuid4())[0:8] + with open(fname, "a") as f: + f.write(container_id) + else: + with open(fname, "r") as f: + container_id = f.read() + + return is_cold, container_id + + +def main(event): + start = datetime.datetime.now().timestamp() + + workflow_name = os.getenv("APPSETTING_WEBSITE_SITE_NAME") + func_name = os.path.basename(os.path.dirname(__file__)) + + module_name = f"{func_name}.{func_name}" + module_path = f"{func_name}/{func_name}.py" + spec = importlib.util.spec_from_file_location(module_name, module_path) + function = importlib.util.module_from_spec(spec) + spec.loader.exec_module(function) + + res = function.handler(event) + + end = datetime.datetime.now().timestamp() + + is_cold, container_id = probe_cold_start() + payload = json.dumps({ + "func": func_name, + "start": start, + "end": end, + "is_cold": is_cold, + "container_id": container_id, + }) + + redis = Redis(host={{REDIS_HOST}}, + port=6379, + decode_responses=True, + socket_connect_timeout=10) + + key = os.path.join(workflow_name, func_name, str(uuid.uuid4())[0:8]) + redis.set(key, payload) + + return res diff --git a/benchmarks/wrappers/azure/python/main_workflow.py b/benchmarks/wrappers/azure/python/main_workflow.py new file mode 100644 index 00000000..154baa93 --- /dev/null +++ b/benchmarks/wrappers/azure/python/main_workflow.py @@ -0,0 +1,58 @@ +import os +import json +import datetime +import uuid + +import azure.functions as func +import azure.durable_functions as df + + +def probe_cold_start(): + is_cold = False + fname = os.path.join("/tmp", "cold_run") + if not os.path.exists(fname): + is_cold = True + container_id = str(uuid.uuid4())[0:8] + with open(fname, "a") as f: + f.write(container_id) + else: + with open(fname, "r") as f: + container_id = f.read() + + return is_cold, container_id + + +async def main(req: func.HttpRequest, starter: str, context: func.Context) -> func.HttpResponse: + event = req.get_json() + + begin = datetime.datetime.now() + + client = df.DurableOrchestrationClient(starter) + instance_id = await client.start_new("run_workflow", None, event) + res = await client.wait_for_completion_or_create_check_status_response(req, instance_id, 1000000) + + end = datetime.datetime.now() + + is_cold, container_id = probe_cold_start() + status = await client.get_status(instance_id) + code = 500 if status.runtime_status == "Failed" else 200 + + try: + result = json.loads(res.get_body()) + except json.decoder.JSONDecodeError: + result = res.get_body().decode() + + body = { + "begin": begin.strftime("%s.%f"), + "end": end.strftime("%s.%f"), + "is_cold": is_cold, + "container_id": container_id, + "request_id": context.invocation_id, + "result": result + } + + return func.HttpResponse( + status_code=code, + body=json.dumps(body), + mimetype="application/json" + ) diff --git a/benchmarks/wrappers/azure/python/run_workflow.py b/benchmarks/wrappers/azure/python/run_workflow.py new file mode 100644 index 00000000..c9e5fa7e --- /dev/null +++ b/benchmarks/wrappers/azure/python/run_workflow.py @@ -0,0 +1,78 @@ +import json +import sys +import os +import operator + +import azure.durable_functions as df + +dir_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(dir_path, os.path.pardir)) + +from .fsm import * + + +def get_var(obj, path: str): + names = path.split(".") + assert(len(names) > 0) + + for n in names: + obj = obj[n] + + return obj + + +def run_workflow(context: df.DurableOrchestrationContext): + with open("definition.json") as f: + definition = json.load(f) + + states = {n: State.deserialize(n, s) + for n, s in definition["states"].items()} + current = states[definition["root"]] + res = context.get_input() + + while current: + if isinstance(current, Task): + res = yield context.call_activity(current.func_name, res) + current = states.get(current.next, None) + elif isinstance(current, Switch): + ops = { + "<": operator.lt, + "<=": operator.le, + "==": operator.eq, + ">=": operator.ge, + ">": operator.gt + } + + next = None + for case in current.cases: + var = get_var(res, case.var) + op = ops[case.op] + if op(var, case.val): + next = states[case.next] + break + + if not next and current.default: + next = states[current.default] + current = next + elif isinstance(current, Map): + array = get_var(res, current.array) + tasks = [context.call_activity(current.func_name, e) for e in array] + res = yield context.task_all(tasks) + + current = states.get(current.next, None) + elif isinstance(current, Repeat): + for i in range(current.count): + res = yield context.call_activity(current.func_name, res) + current = states.get(current.next, None) + elif isinstance(current, Loop): + array = get_var(res, current.array) + for elem in array: + yield context.call_activity(current.func_name, elem) + current = states.get(current.next, None) + else: + raise ValueError(f"Undefined state: {current}") + + return res + + +main = df.Orchestrator.create(run_workflow) \ No newline at end of file diff --git a/benchmarks/wrappers/gcp/python/handler.py b/benchmarks/wrappers/gcp/python/handler_function.py similarity index 100% rename from benchmarks/wrappers/gcp/python/handler.py rename to benchmarks/wrappers/gcp/python/handler_function.py diff --git a/benchmarks/wrappers/gcp/python/handler_workflow.py b/benchmarks/wrappers/gcp/python/handler_workflow.py new file mode 100644 index 00000000..e0dbaa56 --- /dev/null +++ b/benchmarks/wrappers/gcp/python/handler_workflow.py @@ -0,0 +1,59 @@ + +import datetime +import io +import json +import os +import sys +import uuid +import importlib + +# Add current directory to allow location of packages +sys.path.append(os.path.join(os.path.dirname(__file__), '.python_packages/lib/site-packages')) + +from redis import Redis + + +def probe_cold_start(): + is_cold = False + fname = os.path.join("/tmp", "cold_run") + if not os.path.exists(fname): + is_cold = True + container_id = str(uuid.uuid4())[0:8] + with open(fname, "a") as f: + f.write(container_id) + else: + with open(fname, "r") as f: + container_id = f.read() + + return is_cold, container_id + + +def handler(req): + start = datetime.datetime.now().timestamp() + event = req.get_json() + + full_function_name = os.getenv("FUNCTION_NAME") + workflow_name, func_name = full_function_name.split("___") + function = importlib.import_module(f"function.{func_name}") + res = function.handler(event) + + end = datetime.datetime.now().timestamp() + + is_cold, container_id = probe_cold_start() + payload = json.dumps({ + "func": func_name, + "start": start, + "end": end, + "is_cold": is_cold, + "container_id": container_id + }) + + redis = Redis(host={{REDIS_HOST}}, + port=6379, + decode_responses=True, + socket_connect_timeout=10) + + key = os.path.join(workflow_name, func_name, str(uuid.uuid4())[0:8]) + redis.set(key, payload) + + return res diff --git a/config/systems.json b/config/systems.json index 7a3bf450..a43d1513 100644 --- a/config/systems.json +++ b/config/systems.json @@ -32,7 +32,7 @@ } } }, - "aws": { + "aws": { "languages": { "python": { "base_images": { @@ -44,14 +44,14 @@ "images": ["build"], "username": "docker_user", "deployment": { - "files": [ "handler.py", "storage.py"], - "packages": [] + "files": ["handler_function.py", "handler_workflow.py", "storage.py"], + "packages": ["redis"] } }, "nodejs": { "base_images": { "12.x" : "lambci/lambda:build-nodejs12.x", - "10.x" : "lambci/lambda:build-nodejs10.x" + "10.x" : "lambci/lambda:build-nodejs10.x" }, "versions": ["10.x", "12.x"], "images": ["build"], @@ -75,14 +75,14 @@ "images": ["build"], "username": "docker_user", "deployment": { - "files": [ "handler.py", "storage.py"], - "packages": ["azure-storage-blob"] + "files": ["handler_function.py", "main_workflow.py", "handler_workflow.py", "storage.py", "fsm.py", "run_workflow.py"], + "packages": ["azure-storage-blob", "azure-functions", "azure-functions-durable", "redis"] } }, "nodejs": { "base_images": { "10" : "mcr.microsoft.com/azure-functions/node:2.0-node10", - "8" : "mcr.microsoft.com/azure-functions/node:2.0-node8" + "8" : "mcr.microsoft.com/azure-functions/node:2.0-node8" }, "images": ["build"], "username": "docker_user", @@ -110,8 +110,8 @@ "images": ["build"], "username": "docker_user", "deployment": { - "files": [ "handler.py", "storage.py"], - "packages": ["google-cloud-storage"] + "files": ["handler_function.py", "handler_workflow.py", "storage.py"], + "packages": ["google-cloud-storage", "redis"] } }, "nodejs": { diff --git a/docs/design.md b/docs/design.md index 021fe4ec..f39518d9 100644 --- a/docs/design.md +++ b/docs/design.md @@ -47,7 +47,7 @@ on a given platform, in parallel. configuration. `sebs/statistics.py` - implements common statistics routines. - + `sebs/utils.py` - implements serialization and logging configuration used by SeBS. `sebs/faas/` - the abstract interface for all FaaS platforms, see [below](#faas-interface) for details. @@ -87,8 +87,8 @@ used for microarchitectural analysis of local invocations. for the selected platform. Then, an instance of `sebs.Benchmark` is created, and both objects are used to create or update function code package and upload or update input data in the cloud storage with the help of `sebs.faas.PersistentStorage` implementation. -In the end, an object of `sebs.faas.function.Function` is created with exposes a list of triggers -encapsulated in `sebs.faas.function.Trigger`. The function is invoked via a selected trigger, +In the end, an object of `sebs.faas.benchmark.Function` is created with exposes a list of triggers +encapsulated in `sebs.faas.benchmark.Trigger`. The function is invoked via a selected trigger, and the output includes a JSON file with invocation ID and results. `sebs.py benchmark process` - the JSON result from benchmark invocation is read, deserialized, diff --git a/docs/workflows.md b/docs/workflows.md new file mode 100644 index 00000000..963d6d41 --- /dev/null +++ b/docs/workflows.md @@ -0,0 +1,88 @@ +## Workflows + +### Installation + +SeBS makes use of [redis](https://redis.io) in order to make reliable and accurate measurements during the execution of workflows. Ideally, the redis instance should be deployed in the same cloud region such that the write latency is minimal. +Because not all platforms allow connections from a workflow execution to a VPC cache, it proved to be easiest to just deploy a VM and have that machine host redis. Make sure to open port `6379` and admit connections in your VPC accordingly. Redis can be hosted as follows: +```bash +docker run --network=host --name redis -d redis redis-server --save 60 1 --loglevel warning +``` + +### Definition + +All platforms accept different scheduling schemes which makes it cumbersome to run the same tests on different platforms. SeBS defines a workflow scheduling language that is transcribed to the desired platform's scheme. +The schedule is represented by a state machine and is encoded in a JSON file. It starts with the following keys: + +```json +{ + "root": "first_state", + "states": { + } +} +``` + +`root` defines the initial state to start the workflow from, while `states` holds a dictionary of `(name, state)` tuples. The following state types are supported. + +#### Task + +A task state is the most basic state: it executes a serverless function. + +```json +{ + "type": "task", + "func_name": "a_very_useful_func", + "next": "postprocess_the_useful_func" +}, +``` + +`func_name` is the name of the file in the benchmark directory, `next` sets the state with which to follow. + +#### Switch + +A switch state makes it possible to encode basic control flow. + +```json +{ + "type": "switch", + "cases": [ + { + "var": "people.number", + "op": "<", + "val": 10, + "next": "few_people" + }, + { + "var": "people.number", + "op": ">=", + "val": 10, + "next": "many_people" + } + ], + "default": "few_people" +} +``` + +This state transcribes to the following Python expression: +```python +if people.number < 10: + few_people() +elif people.number >= 10: + many_people() +else: + few_people() +``` + +#### Map + +A map state takes a list as input and processes each element in parallel using the given function: + +```json +{ + "type": "map", + "array": "people", + "func_name": "rename_person", + "next": "save" +} +``` + +`array` defines the list to be processed, while `func_name` is the name of the file in the benchmark directory. Note that in contrast to a `task`'s function, this one receives only an element of the given array, not the entire running variable. diff --git a/scripts/run_experiments.py b/scripts/run_experiments.py index c18b96c0..bb29e045 100755 --- a/scripts/run_experiments.py +++ b/scripts/run_experiments.py @@ -507,7 +507,7 @@ def shutdown(self): ''' def create_function(self, code_package: CodePackage, experiment_config :dict): - benchmark = code_package.benchmark + benchmark = code_package.name if code_package.is_cached and code_package.is_cached_valid: func_name = code_package.cached_config['name'] @@ -524,7 +524,7 @@ def create_function(self, code_package: CodePackage, experiment_config :dict): code_location = code_package.code_location # Build code package - package = self.package_code(code_location, code_package.benchmark) + package = self.package_code(code_location, code_package.name) code_size = code_package.recalculate_code_size() cached_cfg = code_package.cached_config @@ -546,13 +546,13 @@ def create_function(self, code_package: CodePackage, experiment_config :dict): code_location = code_package.code_location # Build code package - package = self.package_code(code_location, code_package.benchmark) + package = self.package_code(code_location, code_package.name) code_size = code_package.recalculate_code_size() logging.info('Creating function {fname} in {loc}'.format( fname=func_name, loc=code_location )) - self.cache_client.add_function( + self.cache_client.add_benchmark( deployment='local', benchmark=benchmark, language=self.language, @@ -602,7 +602,7 @@ def create_function(self, code_package: CodePackage, experiment_config :dict): raise RuntimeError('Experiment {} is not supported for language {}!'.format(args.experiment, args.language)) # 2. Locate benchmark - #benchmark_path = find_benchmark(args.benchmark, 'benchmarks') + #benchmark_path = find_package_code(args.benchmark, 'benchmarks') #logging.info('# Located benchmark {} at {}'.format(args.benchmark, benchmark_path)) # 6. Create experiment config diff --git a/sebs.py b/sebs.py index 11329508..740d485f 100755 --- a/sebs.py +++ b/sebs.py @@ -7,16 +7,19 @@ import os import sys import traceback +from time import sleep from typing import cast, Optional import click +import pandas as pd import sebs from sebs import SeBS from sebs.regression import regression_suite -from sebs.utils import update_nested_dict +from sebs.utils import update_nested_dict, download_measurements, connect_to_redis_cache from sebs.faas import System as FaaSSystem -from sebs.faas.function import Trigger +from sebs.faas.benchmark import Trigger +from sebs.local import Local PROJECT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -181,7 +184,7 @@ def benchmark(): help="Override function name for random generation.", ) @common_params -def invoke(benchmark, benchmark_input_size, repetitions, trigger, function_name, **kwargs): +def function(benchmark, benchmark_input_size, repetitions, trigger, function_name, **kwargs): ( config, @@ -198,7 +201,7 @@ def invoke(benchmark, benchmark_input_size, repetitions, trigger, function_name, logging_filename=logging_filename, ) func = deployment_client.get_function( - benchmark_obj, function_name if function_name else deployment_client.default_function_name(benchmark_obj) + benchmark_obj, function_name if function_name else deployment_client.default_benchmark_name(benchmark_obj) ) storage = deployment_client.get_storage( replace_existing=experiment_config.update_storage @@ -220,6 +223,7 @@ def invoke(benchmark, benchmark_input_size, repetitions, trigger, function_name, ) else: trigger = triggers[0] + for i in range(repetitions): sebs_client.logging.info(f"Beginning repetition {i+1}/{repetitions}") ret = trigger.sync_invoke(input_config) @@ -234,6 +238,92 @@ def invoke(benchmark, benchmark_input_size, repetitions, trigger, function_name, out_f.write(sebs.utils.serialize(result)) sebs_client.logging.info("Save results to {}".format(os.path.abspath("experiments.json"))) +@benchmark.command() +@click.argument("benchmark", type=str) # , help="Benchmark to be used.") +@click.argument( + "benchmark-input-size", type=click.Choice(["test", "small", "large"]) +) # help="Input test size") +@click.option( + "--repetitions", default=5, type=int, help="Number of experimental repetitions." +) +@click.option( + "--trigger", + type=click.Choice(["library", "http"]), + default="http", + help="Workflow trigger to be used." +) +@click.option( + "--workflow-name", + default=None, + type=str, + help="Override workflow name for random generation.", +) +@common_params +def workflow(benchmark, benchmark_input_size, repetitions, trigger, workflow_name, **kwargs): + + ( + config, + output_dir, + logging_filename, + sebs_client, + deployment_client, + ) = parse_common_params(**kwargs) + if isinstance(deployment_client, Local): + raise NotImplementedError("Local workflow deployment is currently not supported.") + + redis = connect_to_redis_cache(deployment_client.config.redis_host) + + experiment_config = sebs_client.get_experiment_config(config["experiments"]) + benchmark_obj = sebs_client.get_benchmark( + benchmark, + deployment_client, + experiment_config, + logging_filename=logging_filename, + ) + workflow = deployment_client.get_workflow( + benchmark_obj, workflow_name if workflow_name else deployment_client.default_benchmark_name(benchmark_obj) + ) + storage = deployment_client.get_storage( + replace_existing=experiment_config.update_storage + ) + input_config = benchmark_obj.prepare_input( + storage=storage, size=benchmark_input_size + ) + + measurements = [] + result = sebs.experiments.ExperimentResult( + experiment_config, deployment_client.config + ) + result.begin() + + trigger_type = Trigger.TriggerType.get(trigger) + triggers = workflow.triggers(trigger_type) + if len(triggers) == 0: + trigger = deployment_client.create_trigger( + workflow, trigger_type + ) + else: + trigger = triggers[0] + for i in range(repetitions): + sebs_client.logging.info(f"Beginning repetition {i+1}/{repetitions}") + ret = trigger.sync_invoke(input_config) + if ret.stats.failure: + sebs_client.logging.info(f"Failure on repetition {i+1}/{repetitions}") + + measurements += download_measurements(redis, workflow.name, result.begin_time, rep=i) + result.add_invocation(workflow, ret) + result.end() + + path = os.path.join(output_dir, "results", workflow.name, deployment_client.name()+".csv") + os.makedirs(os.path.dirname(path), exist_ok=True) + + df = pd.DataFrame(measurements) + df.to_csv(path, index=False) + + with open("experiments.json", "w") as out_f: + out_f.write(sebs.utils.serialize(result)) + sebs_client.logging.info("Save results to {}".format(os.path.abspath("experiments.json"))) + @benchmark.command() @common_params @@ -359,7 +449,7 @@ def start(benchmark, benchmark_input_size, output, deployments, remove_container result.add_input(input_config) for i in range(deployments): func = deployment_client.get_function( - benchmark_obj, deployment_client.default_function_name(benchmark_obj) + benchmark_obj, deployment_client.default_benchmark_name(benchmark_obj) ) result.add_function(func) diff --git a/sebs/__init__.py b/sebs/__init__.py index 6eceb356..f347b47b 100644 --- a/sebs/__init__.py +++ b/sebs/__init__.py @@ -4,6 +4,6 @@ # from .azure import * # noqa from .cache import Cache # noqa -from .benchmark import Benchmark # noqa +from .code_package import CodePackage # noqa # from .experiments import * # noqa diff --git a/sebs/aws/aws.py b/sebs/aws/aws.py index 5519bd85..44018154 100644 --- a/sebs/aws/aws.py +++ b/sebs/aws/aws.py @@ -1,5 +1,6 @@ import math import os +import re import shutil import time import uuid @@ -10,13 +11,15 @@ from sebs.aws.s3 import S3 from sebs.aws.function import LambdaFunction +from sebs.aws.workflow import SFNWorkflow +from sebs.aws.generator import SFNGenerator from sebs.aws.config import AWSConfig -from sebs.utils import execute -from sebs.benchmark import Benchmark +from sebs.utils import execute, replace_string_in_file +from sebs.code_package import CodePackage from sebs.cache import Cache from sebs.config import SeBSConfig from sebs.utils import LoggingHandlers -from sebs.faas.function import Function, ExecutionResult, Trigger +from sebs.faas.benchmark import Benchmark, Function, ExecutionResult, Trigger, Workflow from sebs.faas.storage import PersistentStorage from sebs.faas.system import System @@ -38,6 +41,10 @@ def typename(): def function_type() -> "Type[Function]": return LambdaFunction + @staticmethod + def workflow_type() -> "Type[Workflow]": + return SFNWorkflow + @property def config(self) -> AWSConfig: return self._config @@ -68,15 +75,24 @@ def initialize(self, config: Dict[str, str] = {}): aws_secret_access_key=self.config.credentials.secret_key, ) self.get_lambda_client() + self.get_sfn_client() self.get_storage() def get_lambda_client(self): - if not hasattr(self, "client"): - self.client = self.session.client( + if not hasattr(self, "lambda_client"): + self.lambda_client = self.session.client( service_name="lambda", region_name=self.config.region, ) - return self.client + return self.lambda_client + + def get_sfn_client(self): + if not hasattr(self, "stepfunctions_client"): + self.sfn_client = self.session.client( + service_name="stepfunctions", + region_name=self.config.region, + ) + return self.sfn_client """ Create a client instance for cloud storage. When benchmark and buckets @@ -122,13 +138,14 @@ def get_storage(self, replace_existing: bool = False) -> PersistentStorage: benchmark: benchmark name """ - def package_code(self, directory: str, language_name: str, benchmark: str) -> Tuple[str, int]: - + def package_code( + self, code_package: CodePackage, directory: str, is_workflow: bool + ) -> Tuple[str, int]: CONFIG_FILES = { "python": ["handler.py", "requirements.txt", ".python_packages"], "nodejs": ["handler.js", "package.json", "node_modules"], } - package_config = CONFIG_FILES[language_name] + package_config = CONFIG_FILES[code_package.language_name] function_dir = os.path.join(directory, "function") os.makedirs(function_dir) # move all files to 'function' except handler.py @@ -137,42 +154,75 @@ def package_code(self, directory: str, language_name: str, benchmark: str) -> Tu file = os.path.join(directory, file) shutil.move(file, function_dir) + handler_path = os.path.join(directory, CONFIG_FILES[code_package.language_name][0]) + replace_string_in_file(handler_path, "{{REDIS_HOST}}", f'"{self.config.redis_host}"') + + # For python, add an __init__ file + if code_package.language_name == "python": + path = os.path.join(function_dir, "__init__.py") + with open(path, "a"): + os.utime(path, None) + # FIXME: use zipfile # create zip with hidden directory but without parent directory - execute("zip -qu -r9 {}.zip * .".format(benchmark), shell=True, cwd=directory) - benchmark_archive = "{}.zip".format(os.path.join(directory, benchmark)) + execute( + "zip -qu -r9 {}.zip * .".format(code_package.name), + shell=True, + cwd=directory, + ) + benchmark_archive = "{}.zip".format(os.path.join(directory, code_package.name)) self.logging.info("Created {} archive".format(benchmark_archive)) bytes_size = os.path.getsize(os.path.join(directory, benchmark_archive)) mbytes = bytes_size / 1024.0 / 1024.0 self.logging.info("Zip archive size {:2f} MB".format(mbytes)) - return os.path.join(directory, "{}.zip".format(benchmark)), bytes_size + return os.path.join(directory, "{}.zip".format(code_package.name)), bytes_size + + def wait_for_function(self, func_name: str): + ready = False + count = 0 + while not ready: + ret = self.lambda_client.get_function(FunctionName=func_name) + state = ret["Configuration"]["State"] + update_status = ret["Configuration"].get("LastUpdateStatus", "Successful") + ready = (state == "Active") and (update_status == "Successful") + + # If we haven't seen the result yet, wait a second. + if not ready: + count += 1 + time.sleep(10) + elif "Failed" in (state, update_status): + self.logging.error(f"Cannot wait for failed {func_name}") + break - def create_function(self, code_package: Benchmark, func_name: str) -> "LambdaFunction": + if count > 6: + self.logging.error(f"Function {func_name} stuck in state {state} after 60s") + break + def create_function(self, code_package: CodePackage, func_name: str) -> "LambdaFunction": package = code_package.code_location - benchmark = code_package.benchmark + benchmark = code_package.name language = code_package.language_name language_runtime = code_package.language_version - timeout = code_package.benchmark_config.timeout - memory = code_package.benchmark_config.memory + timeout = code_package.config.timeout + memory = code_package.config.memory code_size = code_package.code_size code_bucket: Optional[str] = None - func_name = AWS.format_function_name(func_name) + func_name = AWS.format_resource_name(func_name) storage_client = self.get_storage() # we can either check for exception or use list_functions # there's no API for test try: - ret = self.client.get_function(FunctionName=func_name) + ret = self.lambda_client.get_function(FunctionName=func_name) self.logging.info( "Function {} exists on AWS, retrieve configuration.".format(func_name) ) # Here we assume a single Lambda role lambda_function = LambdaFunction( func_name, - code_package.benchmark, + code_package.name, ret["Configuration"]["FunctionArn"], code_package.hash, timeout, @@ -183,7 +233,7 @@ def create_function(self, code_package: Benchmark, func_name: str) -> "LambdaFun self.update_function(lambda_function, code_package) lambda_function.updated_code = True # TODO: get configuration of REST API - except self.client.exceptions.ResourceNotFoundException: + except self.lambda_client.exceptions.ResourceNotFoundException: self.logging.info("Creating function {} from {}".format(func_name, package)) # AWS Lambda limit on zip deployment size @@ -201,7 +251,7 @@ def create_function(self, code_package: Benchmark, func_name: str) -> "LambdaFun storage_client.upload(code_bucket, package, code_package_name) self.logging.info("Uploading function {} code to {}".format(func_name, code_bucket)) code_config = {"S3Bucket": code_bucket, "S3Key": code_package_name} - ret = self.client.create_function( + ret = self.lambda_client.create_function( FunctionName=func_name, Runtime="{}{}".format(language, language_runtime), Handler="handler.handler", @@ -214,7 +264,7 @@ def create_function(self, code_package: Benchmark, func_name: str) -> "LambdaFun # print(url) lambda_function = LambdaFunction( func_name, - code_package.benchmark, + code_package.name, ret["FunctionArn"], code_package.hash, timeout, @@ -225,22 +275,22 @@ def create_function(self, code_package: Benchmark, func_name: str) -> "LambdaFun ) # Add LibraryTrigger to a new function - from sebs.aws.triggers import LibraryTrigger + from sebs.aws.triggers import FunctionLibraryTrigger - trigger = LibraryTrigger(func_name, self) + trigger = FunctionLibraryTrigger(func_name, self) trigger.logging_handlers = self.logging_handlers lambda_function.add_trigger(trigger) return lambda_function - def cached_function(self, function: Function): + def cached_benchmark(self, benchmark: Benchmark): from sebs.aws.triggers import LibraryTrigger - for trigger in function.triggers(Trigger.TriggerType.LIBRARY): + for trigger in benchmark.triggers(Trigger.TriggerType.LIBRARY): trigger.logging_handlers = self.logging_handlers cast(LibraryTrigger, trigger).deployment_client = self - for trigger in function.triggers(Trigger.TriggerType.HTTP): + for trigger in benchmark.triggers(Trigger.TriggerType.HTTP): trigger.logging_handlers = self.logging_handlers """ @@ -254,49 +304,185 @@ def cached_function(self, function: Function): :param memory: memory limit for function """ - def update_function(self, function: Function, code_package: Benchmark): + def update_function(self, function: Function, code_package: CodePackage): function = cast(LambdaFunction, function) name = function.name code_size = code_package.code_size package = code_package.code_location + # Run AWS update # AWS Lambda limit on zip deployment if code_size < 50 * 1024 * 1024: with open(package, "rb") as code_body: - self.client.update_function_code(FunctionName=name, ZipFile=code_body.read()) + self.lambda_client.update_function_code(FunctionName=name, ZipFile=code_body.read()) # Upload code package to S3, then update else: code_package_name = os.path.basename(package) storage = cast(S3, self.get_storage()) - bucket = function.code_bucket(code_package.benchmark, storage) + bucket = function.code_bucket(code_package.name, storage) storage.upload(bucket, package, code_package_name) - self.client.update_function_code( + self.lambda_client.update_function_code( FunctionName=name, S3Bucket=bucket, S3Key=code_package_name ) - self.logging.info( - f"Updated code of {name} function. " - "Sleep 5 seconds before updating configuration to avoid cloud errors." - ) - time.sleep(5) + + # Wait for code update to finish before updating config + self.wait_for_function(name) + # and update config - self.client.update_function_configuration( + self.lambda_client.update_function_configuration( FunctionName=name, Timeout=function.timeout, MemorySize=function.memory ) self.logging.info("Published new function code") + def create_function_trigger(self, func: Function, trigger_type: Trigger.TriggerType) -> Trigger: + from sebs.aws.triggers import HTTPTrigger + + function = cast(LambdaFunction, func) + + if trigger_type == Trigger.TriggerType.HTTP: + api_name = "{}-http-api".format(function.name) + http_api = self.config.resources.http_api(api_name, function, self.session) + # https://aws.amazon.com/blogs/compute/announcing-http-apis-for-amazon-api-gateway/ + # but this is wrong - source arn must be {api-arn}/*/* + self.get_lambda_client().add_permission( + FunctionName=function.name, + StatementId=str(uuid.uuid1()), + Action="lambda:InvokeFunction", + Principal="apigateway.amazonaws.com", + SourceArn=f"{http_api.arn}/*/*", + ) + trigger = HTTPTrigger(http_api.endpoint, api_name) + trigger.logging_handlers = self.logging_handlers + elif trigger_type == Trigger.TriggerType.LIBRARY: + # should already exist + return func.triggers(Trigger.TriggerType.LIBRARY)[0] + else: + raise RuntimeError("Not supported!") + + function.add_trigger(trigger) + self.cache_client.update_benchmark(function) + return trigger + + def create_workflow(self, code_package: CodePackage, workflow_name: str) -> "SFNWorkflow": + + workflow_name = AWS.format_resource_name(workflow_name) + + # Make sure we have a valid workflow benchmark + definition_path = os.path.join(code_package.path, "definition.json") + if not os.path.exists(definition_path): + raise ValueError(f"No workflow definition found for {workflow_name}") + + # First we create a lambda function for each code file + code_files = list(code_package.get_code_files(include_config=False)) + func_names = [os.path.splitext(os.path.basename(p))[0] for p in code_files] + funcs = [ + self.create_function(code_package, workflow_name + "___" + fn) for fn in func_names + ] + + # Generate workflow definition.json + gen = SFNGenerator({n: f.arn for (n, f) in zip(func_names, funcs)}) + gen.parse(definition_path) + definition = gen.generate() + + package = code_package.code_location + + # We cannot retrieve the state machine because we don't know its ARN + # so we just create it and catch any errors + try: + ret = self.sfn_client.create_state_machine( + name=workflow_name, + definition=definition, + roleArn=self.config.resources.lambda_role(self.session), + ) + + self.logging.info("Creating workflow {} from {}".format(workflow_name, package)) + + workflow = SFNWorkflow( + workflow_name, + funcs, + code_package.name, + ret["stateMachineArn"], + code_package.hash, + ) + except self.sfn_client.exceptions.StateMachineAlreadyExists as e: + match = re.search("'([^']*)'", str(e)) + if not match: + raise + + arn = match.group()[1:-1] + self.logging.info( + "Workflow {} exists on AWS, retrieve configuration.".format(workflow_name) + ) + + # Here we assume a single Lambda role + workflow = SFNWorkflow(workflow_name, funcs, code_package.name, arn, code_package.hash) + + self.update_workflow(workflow, code_package) + workflow.updated_code = True + + # Add LibraryTrigger to a new function + from sebs.aws.triggers import WorkflowLibraryTrigger + + trigger = WorkflowLibraryTrigger(workflow.arn, self) + trigger.logging_handlers = self.logging_handlers + workflow.add_trigger(trigger) + + return workflow + + def update_workflow(self, workflow: Workflow, code_package: CodePackage): + workflow = cast(SFNWorkflow, workflow) + + # Make sure we have a valid workflow benchmark + definition_path = os.path.join(code_package.path, "definition.json") + if not os.path.exists(definition_path): + raise ValueError(f"No workflow definition found for {workflow.name}") + + # Create or update lambda function for each code file + code_files = list(code_package.get_code_files(include_config=False)) + func_names = [os.path.splitext(os.path.basename(p))[0] for p in code_files] + funcs = [ + self.create_function(code_package, workflow.name + "___" + fn) for fn in func_names + ] + + # Generate workflow definition.json + gen = SFNGenerator({n: f.arn for (n, f) in zip(func_names, funcs)}) + gen.parse(definition_path) + definition = gen.generate() + + self.sfn_client.update_state_machine( + stateMachineArn=workflow.arn, + definition=definition, + roleArn=self.config.resources.lambda_role(self.session), + ) + workflow.functions = funcs + self.logging.info("Published new workflow code") + + def create_workflow_trigger( + self, workflow: Workflow, trigger_type: Trigger.TriggerType + ) -> Trigger: + workflow = cast(SFNWorkflow, workflow) + + if trigger_type == Trigger.TriggerType.HTTP: + raise RuntimeError("Not supported!") + elif trigger_type == Trigger.TriggerType.LIBRARY: + # should already exist + return workflow.triggers(Trigger.TriggerType.LIBRARY)[0] + else: + raise RuntimeError("Not supported!") + @staticmethod - def default_function_name(code_package: Benchmark) -> str: + def default_benchmark_name(code_package: CodePackage) -> str: # Create function name func_name = "{}-{}-{}".format( - code_package.benchmark, + code_package.name, code_package.language_name, - code_package.benchmark_config.memory, + code_package.config.memory, ) - return AWS.format_function_name(func_name) + return AWS.format_resource_name(func_name) @staticmethod - def format_function_name(func_name: str) -> str: + def format_resource_name(func_name: str) -> str: # AWS Lambda does not allow hyphens in function names func_name = func_name.replace("-", "_") func_name = func_name.replace(".", "_") @@ -309,7 +495,7 @@ def format_function_name(func_name: str) -> str: def delete_function(self, func_name: Optional[str]): self.logging.debug("Deleting function {}".format(func_name)) try: - self.client.delete_function(FunctionName=func_name) + self.lambda_client.delete_function(FunctionName=func_name) except Exception: self.logging.debug("Function {} does not exist!".format(func_name)) @@ -452,36 +638,6 @@ def download_metrics( f"out of {results_count} invocations" ) - def create_trigger(self, func: Function, trigger_type: Trigger.TriggerType) -> Trigger: - from sebs.aws.triggers import HTTPTrigger - - function = cast(LambdaFunction, func) - - if trigger_type == Trigger.TriggerType.HTTP: - - api_name = "{}-http-api".format(function.name) - http_api = self.config.resources.http_api(api_name, function, self.session) - # https://aws.amazon.com/blogs/compute/announcing-http-apis-for-amazon-api-gateway/ - # but this is wrong - source arn must be {api-arn}/*/* - self.get_lambda_client().add_permission( - FunctionName=function.name, - StatementId=str(uuid.uuid1()), - Action="lambda:InvokeFunction", - Principal="apigateway.amazonaws.com", - SourceArn=f"{http_api.arn}/*/*", - ) - trigger = HTTPTrigger(http_api.endpoint, api_name) - trigger.logging_handlers = self.logging_handlers - elif trigger_type == Trigger.TriggerType.LIBRARY: - # should already exist - return func.triggers(Trigger.TriggerType.LIBRARY)[0] - else: - raise RuntimeError("Not supported!") - - function.add_trigger(trigger) - self.cache_client.update_function(function) - return trigger - def _enforce_cold_start(self, function: Function): func = cast(LambdaFunction, function) self.get_lambda_client().update_function_configuration( @@ -491,7 +647,7 @@ def _enforce_cold_start(self, function: Function): Environment={"Variables": {"ForceColdStart": str(self.cold_start_counter)}}, ) - def enforce_cold_start(self, functions: List[Function], code_package: Benchmark): + def enforce_cold_start(self, functions: List[Function], code_package: CodePackage): self.cold_start_counter += 1 for func in functions: self._enforce_cold_start(func) diff --git a/sebs/aws/config.py b/sebs/aws/config.py index 849f40aa..fa9d0887 100644 --- a/sebs/aws/config.py +++ b/sebs/aws/config.py @@ -119,16 +119,20 @@ def lambda_role(self, boto3_session: boto3.session.Session) -> str: "Version": "2012-10-17", "Statement": [ { + "Sid": "", "Effect": "Allow", - "Principal": {"Service": "lambda.amazonaws.com"}, + "Principal": {"Service": ["lambda.amazonaws.com", "states.amazonaws.com"]}, "Action": "sts:AssumeRole", } ], } - role_name = "sebs-lambda-role" + role_name = "sebs-role" attached_policies = [ "arn:aws:iam::aws:policy/AmazonS3FullAccess", + "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess", "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + "arn:aws:iam::aws:policy/service-role/AWSLambdaRole", + "arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess", ] try: out = iam_client.get_role(RoleName=role_name) @@ -233,10 +237,11 @@ def deserialize(config: dict, cache: Cache, handlers: LoggingHandlers) -> Resour class AWSConfig(Config): - def __init__(self, credentials: AWSCredentials, resources: AWSResources): + def __init__(self, credentials: AWSCredentials, resources: AWSResources, redis_host: str): super().__init__() self._credentials = credentials self._resources = resources + self._redis_host = redis_host @staticmethod def typename() -> str: @@ -250,11 +255,16 @@ def credentials(self) -> AWSCredentials: def resources(self) -> AWSResources: return self._resources + @property + def redis_host(self) -> str: + return self._redis_host + # FIXME: use future annotations (see sebs/faas/system) @staticmethod def initialize(cfg: Config, dct: dict): config = cast(AWSConfig, cfg) config._region = dct["region"] + config._redis_host = dct["redis_host"] @staticmethod def deserialize(config: dict, cache: Cache, handlers: LoggingHandlers) -> Config: @@ -263,7 +273,7 @@ def deserialize(config: dict, cache: Cache, handlers: LoggingHandlers) -> Config # FIXME: use future annotations (see sebs/faas/system) credentials = cast(AWSCredentials, AWSCredentials.deserialize(config, cache, handlers)) resources = cast(AWSResources, AWSResources.deserialize(config, cache, handlers)) - config_obj = AWSConfig(credentials, resources) + config_obj = AWSConfig(credentials, resources, cached_config["redis_host"]) config_obj.logging_handlers = handlers # Load cached values if cached_config: @@ -294,5 +304,6 @@ def serialize(self) -> dict: "region": self._region, "credentials": self._credentials.serialize(), "resources": self._resources.serialize(), + "redis_host": self._redis_host, } return out diff --git a/sebs/aws/function.py b/sebs/aws/function.py index 36b52c27..20816745 100644 --- a/sebs/aws/function.py +++ b/sebs/aws/function.py @@ -1,7 +1,7 @@ from typing import cast, Optional from sebs.aws.s3 import S3 -from sebs.faas.function import Function +from sebs.faas.benchmark import Function class LambdaFunction(Function): @@ -42,12 +42,12 @@ def serialize(self) -> dict: @staticmethod def deserialize(cached_config: dict) -> "LambdaFunction": - from sebs.faas.function import Trigger - from sebs.aws.triggers import LibraryTrigger, HTTPTrigger + from sebs.faas.benchmark import Trigger + from sebs.aws.triggers import FunctionLibraryTrigger, HTTPTrigger ret = LambdaFunction( cached_config["name"], - cached_config["benchmark"], + cached_config["code_package"], cached_config["arn"], cached_config["hash"], cached_config["timeout"], @@ -59,7 +59,7 @@ def deserialize(cached_config: dict) -> "LambdaFunction": for trigger in cached_config["triggers"]: trigger_type = cast( Trigger, - {"Library": LibraryTrigger, "HTTP": HTTPTrigger}.get(trigger["type"]), + {"Library": FunctionLibraryTrigger, "HTTP": HTTPTrigger}.get(trigger["type"]), ) assert trigger_type, "Unknown trigger type {}".format(trigger["type"]) ret.add_trigger(trigger_type.deserialize(trigger)) diff --git a/sebs/aws/generator.py b/sebs/aws/generator.py new file mode 100644 index 00000000..4bc38ee7 --- /dev/null +++ b/sebs/aws/generator.py @@ -0,0 +1,97 @@ +from typing import Dict, List, Union, Any +import numbers +import uuid + +from sebs.faas.fsm import Generator, State, Task, Switch, Map, Repeat, Loop + + +class SFNGenerator(Generator): + def __init__(self, func_arns: Dict[str, str]): + super().__init__() + self._func_arns = func_arns + + def postprocess(self, payloads: List[dict]) -> dict: + def _nameless(p: dict) -> dict: + del p["Name"] + return p + + state_payloads = {p["Name"]: _nameless(p) for p in payloads} + definition = { + "Comment": "SeBS auto-generated benchmark", + "StartAt": self.root.name, + "States": state_payloads, + } + + return definition + + def encode_task(self, state: Task) -> Union[dict, List[dict]]: + payload: Dict[str, Any] = { + "Name": state.name, + "Type": "Task", + "Resource": self._func_arns[state.func_name] + } + + if state.next: + payload["Next"] = state.next + else: + payload["End"] = True + + return payload + + def encode_switch(self, state: Switch) -> Union[dict, List[dict]]: + choises = [self._encode_case(c) for c in state.cases] + return { + "Name": state.name, + "Type": "Choice", + "Choices": choises, + "Default": state.default + } + + def _encode_case(self, case: Switch.Case) -> dict: + type = "Numeric" if isinstance(case.val, numbers.Number) else "String" + comp = { + "<": "LessThan", + "<=": "LessThanEquals", + "==": "Equals", + ">=": "GreaterThanEquals", + ">": "GreaterThan", + } + cond = type + comp[case.op] + + return {"Variable": "$." + case.var, cond: case.val, "Next": case.next} + + def encode_map(self, state: Map) -> Union[dict, List[dict]]: + map_func_name = "func_" + str(uuid.uuid4())[:8] + + payload: Dict[str, Any] = { + "Name": state.name, + "Type": "Map", + "ItemsPath": "$." + state.array, + "Iterator": { + "StartAt": map_func_name, + "States": { + map_func_name: { + "Type": "Task", + "Resource": self._func_arns[state.func_name], + "End": True, + } + }, + }, + } + + if state.next: + payload["Next"] = state.next + else: + payload["End"] = True + + return payload + + def encode_loop(self, state: Loop) -> Union[dict, List[dict]]: + map_state = Map(self.name, self.func_name, self.array, self.next) + payload = self.encode_map(map_state) + payload["MaxConcurrency"] = 1 + payload["ResultSelector"] = dict() + payload["ResultPath"] = "$." + str(uuid.uuid4())[:8] + + return payload + diff --git a/sebs/aws/triggers.py b/sebs/aws/triggers.py index f1831459..0a5332f0 100644 --- a/sebs/aws/triggers.py +++ b/sebs/aws/triggers.py @@ -2,10 +2,11 @@ import concurrent.futures import datetime import json +import time from typing import Dict, Optional # noqa from sebs.aws.aws import AWS -from sebs.faas.function import ExecutionResult, Trigger +from sebs.faas.benchmark import ExecutionResult, Trigger class LibraryTrigger(Trigger): @@ -31,10 +32,21 @@ def deployment_client(self, deployment_client: AWS): def trigger_type() -> Trigger.TriggerType: return Trigger.TriggerType.LIBRARY + def serialize(self) -> dict: + return {"type": "Library", "name": self.name} + + @classmethod + def deserialize(cls, obj: dict) -> Trigger: + return cls(obj["name"]) + + +class FunctionLibraryTrigger(LibraryTrigger): def sync_invoke(self, payload: dict) -> ExecutionResult: self.logging.debug(f"Invoke function {self.name}") + self.deployment_client.wait_for_function(self.name) + serialized_payload = json.dumps(payload).encode("utf-8") client = self.deployment_client.get_lambda_client() begin = datetime.datetime.now() @@ -84,12 +96,42 @@ def async_invoke(self, payload: dict): raise RuntimeError() return ret - def serialize(self) -> dict: - return {"type": "Library", "name": self.name} - @staticmethod - def deserialize(obj: dict) -> Trigger: - return LibraryTrigger(obj["name"]) +class WorkflowLibraryTrigger(LibraryTrigger): + def sync_invoke(self, payload: dict) -> ExecutionResult: + + self.logging.debug(f"Invoke workflow {self.name}") + + client = self.deployment_client.get_sfn_client() + begin = datetime.datetime.now() + ret = client.start_execution(stateMachineArn=self.name, input=json.dumps(payload)) + end = datetime.datetime.now() + + aws_result = ExecutionResult.from_times(begin, end) + aws_result.request_id = ret["ResponseMetadata"]["RequestId"] + execution_arn = ret["executionArn"] + + # Wait for execution to finish, then print results. + execution_finished = False + while not execution_finished: + execution = client.describe_execution(executionArn=execution_arn) + status = execution["status"] + execution_finished = status != "RUNNING" + + # If we haven't seen the result yet, wait a second. + if not execution_finished: + time.sleep(10) + elif status == "FAILED": + self.logging.error(f"Invocation of {self.name} failed") + self.logging.error(f"Input: {payload}") + aws_result.stats.failure = True + return aws_result + + return aws_result + + def async_invoke(self, payload: dict): + + raise NotImplementedError("Async invocation is not implemented") class HTTPTrigger(Trigger): diff --git a/sebs/aws/workflow.py b/sebs/aws/workflow.py new file mode 100644 index 00000000..ac48dc4e --- /dev/null +++ b/sebs/aws/workflow.py @@ -0,0 +1,56 @@ +from typing import cast, List + +from sebs.aws.s3 import S3 +from sebs.aws.function import LambdaFunction +from sebs.faas.benchmark import Workflow + + +class SFNWorkflow(Workflow): + def __init__( + self, + name: str, + functions: List[LambdaFunction], + benchmark: str, + arn: str, + code_package_hash: str, + ): + super().__init__(benchmark, name, code_package_hash) + self.functions = functions + self.arn = arn + + @staticmethod + def typename() -> str: + return "AWS.SFNWorkflow" + + def serialize(self) -> dict: + return { + **super().serialize(), + "functions": [f.serialize() for f in self.functions], + "arn": self.arn, + } + + @staticmethod + def deserialize(cached_config: dict) -> "SFNWorkflow": + from sebs.faas.benchmark import Trigger + from sebs.aws.triggers import WorkflowLibraryTrigger, HTTPTrigger + + funcs = [LambdaFunction.deserialize(f) for f in cached_config["functions"]] + ret = SFNWorkflow( + cached_config["name"], + funcs, + cached_config["code_package"], + cached_config["arn"], + cached_config["hash"], + ) + for trigger in cached_config["triggers"]: + trigger_type = cast( + Trigger, + {"Library": WorkflowLibraryTrigger, "HTTP": HTTPTrigger}.get(trigger["type"]), + ) + assert trigger_type, "Unknown trigger type {}".format(trigger["type"]) + ret.add_trigger(trigger_type.deserialize(trigger)) + return ret + + def code_bucket(self, benchmark: str, storage_client: S3): + self.bucket, idx = storage_client.add_input_bucket(benchmark) + return self.bucket diff --git a/sebs/azure/__init__.py b/sebs/azure/__init__.py index 499b1372..5736abdc 100644 --- a/sebs/azure/__init__.py +++ b/sebs/azure/__init__.py @@ -1,4 +1,4 @@ from .azure import Azure # noqa -from .function import AzureFunction # noqa +from .function_app import AzureFunction, AzureWorkflow # noqa from .config import AzureConfig # noqa from .blob_storage import BlobStorage # noqa diff --git a/sebs/azure/azure.py b/sebs/azure/azure.py index a12289e4..3fb3fcbd 100644 --- a/sebs/azure/azure.py +++ b/sebs/azure/azure.py @@ -1,25 +1,25 @@ import datetime import json +import glob import os import shutil import time -from typing import cast, Dict, List, Optional, Set, Tuple, Type # noqa +from typing import cast, Dict, List, Optional, Set, Tuple, Type, TypeVar # noqa import docker from sebs.azure.blob_storage import BlobStorage from sebs.azure.cli import AzureCLI -from sebs.azure.function import AzureFunction +from sebs.azure.function_app import FunctionApp, AzureFunction, AzureWorkflow from sebs.azure.config import AzureConfig, AzureResources from sebs.azure.triggers import AzureTrigger, HTTPTrigger -from sebs.faas.function import Trigger -from sebs.benchmark import Benchmark +from sebs.code_package import CodePackage from sebs.cache import Cache from sebs.config import SeBSConfig -from sebs.utils import LoggingHandlers, execute -from ..faas.function import Function, ExecutionResult -from ..faas.storage import PersistentStorage -from ..faas.system import System +from sebs.utils import LoggingHandlers, execute, replace_string_in_file +from sebs.faas.benchmark import Benchmark, Function, ExecutionResult, Workflow, Trigger +from sebs.faas.storage import PersistentStorage +from sebs.faas.system import System class Azure(System): @@ -43,6 +43,10 @@ def config(self) -> AzureConfig: def function_type() -> Type[Function]: return AzureFunction + @staticmethod + def workflow_type() -> Type[Workflow]: + return AzureWorkflow + def __init__( self, sebs_config: SeBSConfig, @@ -114,82 +118,137 @@ def get_storage(self, replace_existing: bool = False) -> PersistentStorage: # - function.json # host.json # requirements.txt/package.json - def package_code(self, directory: str, language_name: str, benchmark: str) -> Tuple[str, int]: + def package_code( + self, code_package: CodePackage, directory: str, is_workflow: bool + ) -> Tuple[str, int]: # In previous step we ran a Docker container which installed packages # Python packages are in .python_packages because this is expected by Azure - EXEC_FILES = {"python": "handler.py", "nodejs": "handler.js"} + FILES = {"python": "*.py", "nodejs": "*.js"} CONFIG_FILES = { "python": ["requirements.txt", ".python_packages"], "nodejs": ["package.json", "node_modules"], } - package_config = CONFIG_FILES[language_name] + WRAPPER_FILES = { + "python": ["handler.py", "storage.py", "fsm.py"], + "nodejs": ["handler.js", "storage.js"], + } + file_type = FILES[code_package.language_name] + package_config = CONFIG_FILES[code_package.language_name] + wrapper_files = WRAPPER_FILES[code_package.language_name] - handler_dir = os.path.join(directory, "handler") - os.makedirs(handler_dir) - # move all files to 'handler' except package config - for f in os.listdir(directory): - if f not in package_config: - source_file = os.path.join(directory, f) - shutil.move(source_file, handler_dir) + main_path = os.path.join(directory, "main_workflow.py") + if is_workflow: + os.rename(main_path, os.path.join(directory, "main.py")) + + # Make sure we have a valid workflow benchmark + src_path = os.path.join(code_package.path, "definition.json") + if not os.path.exists(src_path): + raise ValueError(f"No workflow definition found in {directory}") + + dst_path = os.path.join(directory, "definition.json") + shutil.copy2(src_path, dst_path) + else: + os.remove(main_path) - # generate function.json # TODO: extension to other triggers than HTTP - default_function_json = { - "scriptFile": EXEC_FILES[language_name], - "bindings": [ - { - "authLevel": "function", - "type": "httpTrigger", - "direction": "in", - "name": "req", - "methods": ["get", "post"], - }, - {"type": "http", "direction": "out", "name": "$return"}, - ], - } - json_out = os.path.join(directory, "handler", "function.json") - json.dump(default_function_json, open(json_out, "w"), indent=2) + main_bindings = [ + { + "name": "req", + "type": "httpTrigger", + "direction": "in", + "authLevel": "function", + "methods": ["post"], + }, + {"name": "starter", "type": "durableClient", "direction": "in"}, + {"name": "$return", "type": "http", "direction": "out"}, + ] + activity_bindings = [ + {"name": "event", "type": "activityTrigger", "direction": "in"}, + ] + orchestrator_bindings = [ + {"name": "context", "type": "orchestrationTrigger", "direction": "in"} + ] + + if is_workflow: + bindings = {"main": main_bindings, "run_workflow": orchestrator_bindings} + else: + bindings = {"function": main_bindings} + + func_dirs = [] + for file_path in glob.glob(os.path.join(directory, file_type)): + file = os.path.basename(file_path) + + if file in package_config or file in wrapper_files: + continue + + # move file directory/f.py to directory/f/f.py + name, ext = os.path.splitext(file) + func_dir = os.path.join(directory, name) + func_dirs.append(func_dir) + + dst_file = os.path.join(func_dir, file) + src_file = os.path.join(directory, file) + os.makedirs(func_dir) + shutil.move(src_file, dst_file) + + # generate function.json + script_file = file if (name in bindings and is_workflow) else "handler.py" + payload = { + "bindings": bindings.get(name, activity_bindings), + "scriptFile": script_file, + "disabled": False, + } + dst_json = os.path.join(os.path.dirname(dst_file), "function.json") + json.dump(payload, open(dst_json, "w"), indent=2) + + handler_path = os.path.join(directory, WRAPPER_FILES[code_package.language_name][0]) + replace_string_in_file(handler_path, "{{REDIS_HOST}}", f'"{self.config.redis_host}"') + + # copy every wrapper file to respective function dirs + for wrapper_file in wrapper_files: + src_path = os.path.join(directory, wrapper_file) + for func_dir in func_dirs: + dst_path = os.path.join(func_dir, wrapper_file) + shutil.copyfile(src_path, dst_path) + os.remove(src_path) # generate host.json - default_host_json = { + host_json = { "version": "2.0", "extensionBundle": { "id": "Microsoft.Azure.Functions.ExtensionBundle", - "version": "[1.*, 2.0.0)", + "version": "[2.*, 3.0.0)", }, } - json.dump(default_host_json, open(os.path.join(directory, "host.json"), "w"), indent=2) + json.dump(host_json, open(os.path.join(directory, "host.json"), "w"), indent=2) - code_size = Benchmark.directory_size(directory) - execute("zip -qu -r9 {}.zip * .".format(benchmark), shell=True, cwd=directory) + code_size = CodePackage.directory_size(directory) + execute( + "zip -qu -r9 {}.zip * .".format(code_package.name), + shell=True, + cwd=directory, + ) return directory, code_size - def publish_function( + def publish_benchmark( self, - function: Function, - code_package: Benchmark, + benchmark: Benchmark, + code_package: CodePackage, repeat_on_failure: bool = False, ) -> str: success = False url = "" - self.logging.info("Attempting publish of function {}".format(function.name)) + self.logging.info("Attempting publish of {}".format(benchmark.name)) while not success: try: ret = self.cli_instance.execute( "bash -c 'cd /mnt/function " "&& func azure functionapp publish {} --{} --no-build'".format( - function.name, self.AZURE_RUNTIMES[code_package.language_name] + benchmark.name, self.AZURE_RUNTIMES[code_package.language_name] ) ) - # ret = self.cli_instance.execute( - # "bash -c 'cd /mnt/function " - # "&& az functionapp deployment source config-zip " - # "--src {}.zip -g {} -n {} --build-remote false '".format( - # code_package.benchmark, resource_group, function.name - # ) - # ) - # print(ret) + url = "" for line in ret.split(b"\n"): line = line.decode("utf-8") @@ -208,7 +267,7 @@ def publish_function( time.sleep(30) self.logging.info( "Sleep 30 seconds for Azure to register function app {}".format( - function.name + benchmark.name ) ) # escape loop. we failed! @@ -227,26 +286,26 @@ def publish_function( :return: URL to reach HTTP-triggered function """ - def update_function(self, function: Function, code_package: Benchmark): + def update_benchmark(self, benchmark: Benchmark, code_package: CodePackage): # Mount code package in Docker instance self._mount_function_code(code_package) - url = self.publish_function(function, code_package, True) + url = self.publish_benchmark(benchmark, code_package, True) trigger = HTTPTrigger(url, self.config.resources.data_storage_account(self.cli_instance)) trigger.logging_handlers = self.logging_handlers - function.add_trigger(trigger) + benchmark.add_trigger(trigger) - def _mount_function_code(self, code_package: Benchmark): + def _mount_function_code(self, code_package: CodePackage): self.cli_instance.upload_package(code_package.code_location, "/mnt/function/") - def default_function_name(self, code_package: Benchmark) -> str: + def default_benchmark_name(self, code_package: CodePackage) -> str: """ Functionapp names must be globally unique in Azure. """ func_name = ( "{}-{}-{}".format( - code_package.benchmark, + code_package.name, code_package.language_name, self.config.resources_id, ) @@ -255,8 +314,9 @@ def default_function_name(self, code_package: Benchmark) -> str: ) return func_name - def create_function(self, code_package: Benchmark, func_name: str) -> AzureFunction: + B = TypeVar("B", bound=FunctionApp) + def create_benchmark(self, code_package: CodePackage, name: str, benchmark_cls: Type[B]) -> B: language = code_package.language_name language_runtime = code_package.language_version resource_group = self.config.resources.resource_group(self.cli_instance) @@ -264,7 +324,7 @@ def create_function(self, code_package: Benchmark, func_name: str) -> AzureFunct config = { "resource_group": resource_group, - "func_name": func_name, + "name": name, "region": region, "runtime": self.AZURE_RUNTIMES[language], "runtime_version": language_runtime, @@ -277,7 +337,7 @@ def create_function(self, code_package: Benchmark, func_name: str) -> AzureFunct ( " az functionapp config appsettings list " " --resource-group {resource_group} " - " --name {func_name} " + " --name {name} " ).format(**config) ) for setting in json.loads(ret.decode()): @@ -288,7 +348,7 @@ def create_function(self, code_package: Benchmark, func_name: str) -> AzureFunct function_storage_account = AzureResources.Storage.from_cache( account_name, connection_string ) - self.logging.info("Azure: Selected {} function app".format(func_name)) + self.logging.info("Azure: Selected {} function app".format(name)) except RuntimeError: function_storage_account = self.config.resources.add_storage_account(self.cli_instance) config["storage_account"] = function_storage_account.account_name @@ -298,49 +358,54 @@ def create_function(self, code_package: Benchmark, func_name: str) -> AzureFunct # create function app self.cli_instance.execute( ( - " az functionapp create --resource-group {resource_group} " - " --os-type Linux --consumption-plan-location {region} " + " az functionapp create --functions-version 3 " + " --resource-group {resource_group} --os-type Linux" + " --consumption-plan-location {region} " " --runtime {runtime} --runtime-version {runtime_version} " - " --name {func_name} --storage-account {storage_account}" + " --name {name} --storage-account {storage_account}" ).format(**config) ) - self.logging.info("Azure: Created function app {}".format(func_name)) + self.logging.info("Azure: Created function app {}".format(name)) break except RuntimeError as e: # Azure does not allow some concurrent operations if "another operation is in progress" in str(e): - self.logging.info( - f"Repeat {func_name} creation, another operation in progress" - ) + self.logging.info(f"Repeat {name} creation, another operation in progress") # Rethrow -> another error else: raise - function = AzureFunction( - name=func_name, - benchmark=code_package.benchmark, + benchmark = benchmark_cls( + name=name, + benchmark=code_package.name, code_hash=code_package.hash, function_storage=function_storage_account, ) # update existing function app - self.update_function(function, code_package) + self.update_benchmark(benchmark, code_package) - self.cache_client.add_function( - deployment_name=self.name(), - language_name=language, - code_package=code_package, - function=function, - ) - return function + return benchmark - def cached_function(self, function: Function): + def cached_benchmark(self, benchmark: Benchmark): data_storage_account = self.config.resources.data_storage_account(self.cli_instance) - for trigger in function.triggers_all(): + for trigger in benchmark.triggers_all(): azure_trigger = cast(AzureTrigger, trigger) azure_trigger.logging_handlers = self.logging_handlers azure_trigger.data_storage_account = data_storage_account + def create_function(self, code_package: CodePackage, func_name: str) -> AzureFunction: + return self.create_benchmark(code_package, func_name, AzureFunction) + + def update_function(self, function: Function, code_package: CodePackage): + self.update_benchmark(function, code_package) + + def create_workflow(self, code_package: CodePackage, workflow_name: str) -> AzureWorkflow: + return self.create_benchmark(code_package, workflow_name, AzureWorkflow) + + def update_workflow(self, workflow: Workflow, code_package: CodePackage): + self.update_benchmark(workflow, code_package) + """ Prepare Azure resources to store experiment results. Allocate one container. @@ -431,7 +496,7 @@ def download_metrics( # TODO: query performance counters for mem - def _enforce_cold_start(self, function: Function, code_package: Benchmark): + def _enforce_cold_start(self, function: Function, code_package: CodePackage): fname = function.name resource_group = self.config.resources.resource_group(self.cli_instance) @@ -442,9 +507,9 @@ def _enforce_cold_start(self, function: Function, code_package: Benchmark): f" --settings ForceColdStart={self.cold_start_counter}" ) - self.update_function(function, code_package) + self.update_benchmark(function, code_package) - def enforce_cold_start(self, functions: List[Function], code_package: Benchmark): + def enforce_cold_start(self, functions: List[Function], code_package: CodePackage): self.cold_start_counter += 1 for func in functions: self._enforce_cold_start(func, code_package) @@ -457,67 +522,12 @@ def enforce_cold_start(self, functions: List[Function], code_package: Benchmark) It is automatically created for each function. """ - def create_trigger(self, function: Function, trigger_type: Trigger.TriggerType) -> Trigger: + def create_function_trigger( + self, function: Function, trigger_type: Trigger.TriggerType + ) -> Trigger: raise NotImplementedError() - -# -# def create_azure_function(self, fname, config): -# -# # create function name -# region = self.config["config"]["region"] -# # only hyphens are allowed -# # and name needs to be globally unique -# func_name = fname.replace(".", "-").replace("_", "-") -# -# # create function app -# self.cli_instance.execute( -# ( -# "az functionapp create --resource-group {} " -# "--os-type Linux --consumption-plan-location {} " -# "--runtime {} --runtime-version {} --name {} " -# "--storage-account {}" -# ).format( -# self.resource_group_name, -# region, -# self.AZURE_RUNTIMES[self.language], -# self.config["config"]["runtime"][self.language], -# func_name, -# self.storage_account_name, -# ) -# ) -# logging.info("Created function app {}".format(func_name)) -# return func_name -# -# init = False -# -# def create_function_copies( -# self, -# function_names: List[str], -# code_package: Benchmark, -# experiment_config: dict, -# ): -# -# if not self.init: -# code_location = code_package.code_location -# # package = self.package_code(code_location, code_package.benchmark) -# # code_size = code_package.code_size -# # Restart Docker instance to make sure code package is mounted -# self.start(code_location, restart=True) -# self.storage_account() -# self.resource_group() -# self.init = True -# -# # names = [] -# # for fname in function_names: -# # names.append(self.create_azure_function(fname, experiment_config)) -# names = function_names -# -# # time.sleep(30) -# urls = [] -# for fname in function_names: -# url = self.publish_function(fname, repeat_on_failure=True) -# urls.append(url) -# logging.info("Published function app {} with URL {}".format(fname, url)) -# -# return names, urls + def create_workflow_trigger( + self, workflow: Workflow, trigger_type: Trigger.TriggerType + ) -> Trigger: + raise NotImplementedError() diff --git a/sebs/azure/config.py b/sebs/azure/config.py index 20591595..ebdd2e87 100644 --- a/sebs/azure/config.py +++ b/sebs/azure/config.py @@ -269,11 +269,12 @@ def deserialize(config: dict, cache: Cache, handlers: LoggingHandlers) -> Resour class AzureConfig(Config): - def __init__(self, credentials: AzureCredentials, resources: AzureResources): + def __init__(self, credentials: AzureCredentials, resources: AzureResources, redis_host: str): super().__init__() self._resources_id = "" self._credentials = credentials self._resources = resources + self._redis_host = redis_host @property def credentials(self) -> AzureCredentials: @@ -287,11 +288,16 @@ def resources(self) -> AzureResources: def resources_id(self) -> str: return self._resources_id + @property + def redis_host(self) -> str: + return self._redis_host + # FIXME: use future annotations (see sebs/faas/system) @staticmethod def initialize(cfg: Config, dct: dict): config = cast(AzureConfig, cfg) config._region = dct["region"] + config._redis_host = dct["redis_host"] if "resources_id" in dct: config._resources_id = dct["resources_id"] else: @@ -308,7 +314,7 @@ def deserialize(config: dict, cache: Cache, handlers: LoggingHandlers) -> Config # FIXME: use future annotations (see sebs/faas/system) credentials = cast(AzureCredentials, AzureCredentials.deserialize(config, cache, handlers)) resources = cast(AzureResources, AzureResources.deserialize(config, cache, handlers)) - config_obj = AzureConfig(credentials, resources) + config_obj = AzureConfig(credentials, resources, cached_config["redis_host"]) config_obj.logging_handlers = handlers # Load cached values if cached_config: @@ -341,5 +347,6 @@ def serialize(self) -> dict: "resources_id": self.resources_id, "credentials": self._credentials.serialize(), "resources": self._resources.serialize(), + "redis_host": self._redis_host, } return out diff --git a/sebs/azure/function.py b/sebs/azure/function_app.py similarity index 60% rename from sebs/azure/function.py rename to sebs/azure/function_app.py index ade7e980..0f62c400 100644 --- a/sebs/azure/function.py +++ b/sebs/azure/function_app.py @@ -1,8 +1,10 @@ from sebs.azure.config import AzureResources -from sebs.faas.function import Function +from sebs.faas.benchmark import Benchmark, Function, Workflow +from typing import cast -class AzureFunction(Function): + +class FunctionApp(Benchmark): def __init__( self, name: str, @@ -20,10 +22,10 @@ def serialize(self) -> dict: } @staticmethod - def deserialize(cached_config: dict) -> Function: - ret = AzureFunction( + def deserialize(cached_config: dict) -> "FunctionApp": + ret = FunctionApp( cached_config["name"], - cached_config["benchmark"], + cached_config["code_package"], cached_config["hash"], AzureResources.Storage.deserialize(cached_config["function_storage"]), ) @@ -34,3 +36,15 @@ def deserialize(cached_config: dict) -> Function: assert trigger_type, "Unknown trigger type {}".format(trigger["type"]) ret.add_trigger(trigger_type.deserialize(trigger)) return ret + + +class AzureFunction(Function, FunctionApp): + @staticmethod + def deserialize(cached_config: dict) -> "AzureFunction": + return cast(AzureFunction, FunctionApp.deserialize(cached_config)) + + +class AzureWorkflow(Workflow, FunctionApp): + @staticmethod + def deserialize(cached_config: dict) -> "AzureWorkflow": + return cast(AzureWorkflow, FunctionApp.deserialize(cached_config)) diff --git a/sebs/azure/triggers.py b/sebs/azure/triggers.py index 66be8c6d..a0c8bfdc 100644 --- a/sebs/azure/triggers.py +++ b/sebs/azure/triggers.py @@ -2,7 +2,7 @@ from typing import Any, Dict, Optional # noqa from sebs.azure.config import AzureResources -from sebs.faas.function import ExecutionResult, Trigger +from sebs.faas.benchmark import ExecutionResult, Trigger class AzureTrigger(Trigger): @@ -30,7 +30,6 @@ def trigger_type() -> Trigger.TriggerType: return Trigger.TriggerType.HTTP def sync_invoke(self, payload: dict) -> ExecutionResult: - payload["connection_string"] = self.data_storage_account.connection_string return self._http_invoke(payload, self.url) @@ -42,6 +41,6 @@ def async_invoke(self, payload: dict) -> concurrent.futures.Future: def serialize(self) -> dict: return {"type": "HTTP", "url": self.url} - @staticmethod - def deserialize(obj: dict) -> Trigger: + @classmethod + def deserialize(cls, obj: dict) -> Trigger: return HTTPTrigger(obj["url"]) diff --git a/sebs/cache.py b/sebs/cache.py index dcce8ff7..dc6fe536 100644 --- a/sebs/cache.py +++ b/sebs/cache.py @@ -10,8 +10,8 @@ from sebs.utils import LoggingBase if TYPE_CHECKING: - from sebs.benchmark import Benchmark - from sebs.faas.function import Function + from sebs.code_package import CodePackage + from sebs.faas.benchmark import Benchmark def update(d, u): @@ -45,7 +45,7 @@ class Cache(LoggingBase): def __init__(self, cache_dir: str): super().__init__() self.cache_dir = os.path.abspath(cache_dir) - self.ignore_functions: bool = False + self.ignore_benchmarks: bool = False self.ignore_storage: bool = False self._lock = threading.RLock() if not os.path.exists(self.cache_dir): @@ -55,7 +55,7 @@ def __init__(self, cache_dir: str): @staticmethod def typename() -> str: - return "Benchmark" + return "CodePackage" def load_config(self): with self._lock: @@ -129,12 +129,12 @@ def get_code_package( else: return None - def get_functions( + def get_benchmarks( self, deployment: str, benchmark: str, language: str ) -> Optional[Dict[str, Any]]: cfg = self.get_benchmark_config(deployment, benchmark) - if cfg and language in cfg and not self.ignore_functions: - return cfg[language]["functions"] + if cfg and language in cfg and not self.ignore_benchmarks: + return cfg[language]["benchmarks"] else: return None @@ -162,10 +162,12 @@ def update_storage(self, deployment: str, benchmark: str, config: dict): with open(os.path.join(benchmark_dir, "config.json"), "w") as fp: json.dump(cached_config, fp, indent=2) - def add_code_package(self, deployment_name: str, language_name: str, code_package: "Benchmark"): + def add_code_package( + self, deployment_name: str, language_name: str, code_package: "CodePackage" + ): with self._lock: language = code_package.language_name - benchmark_dir = os.path.join(self.cache_dir, code_package.benchmark) + benchmark_dir = os.path.join(self.cache_dir, code_package.name) os.makedirs(benchmark_dir, exist_ok=True) # Check if cache directory for this deployment exist cached_dir = os.path.join(benchmark_dir, deployment_name, language) @@ -183,7 +185,7 @@ def add_code_package(self, deployment_name: str, language_name: str, code_packag shutil.copy2(code_package.code_location, cached_dir) language_config: Dict[str, Any] = { "code_package": code_package.serialize(), - "functions": {}, + "benchmarks": {}, } # don't store absolute path to avoid problems with moving cache dir relative_cached_loc = os.path.relpath(cached_location, self.cache_dir) @@ -211,16 +213,16 @@ def add_code_package(self, deployment_name: str, language_name: str, code_packag # TODO: update raise RuntimeError( "Cached application {} for {} already exists!".format( - code_package.benchmark, deployment_name + code_package.name, deployment_name ) ) def update_code_package( - self, deployment_name: str, language_name: str, code_package: "Benchmark" + self, deployment_name: str, language_name: str, code_package: "CodePackage" ): with self._lock: language = code_package.language_name - benchmark_dir = os.path.join(self.cache_dir, code_package.benchmark) + benchmark_dir = os.path.join(self.cache_dir, code_package.name) # Check if cache directory for this deployment exist cached_dir = os.path.join(benchmark_dir, deployment_name, language) if os.path.exists(cached_dir): @@ -250,7 +252,7 @@ def update_code_package( self.add_code_package(deployment_name, language_name, code_package) """ - Add new function to cache. + Add new benchmark to cache. :param deployment: :param benchmark: @@ -260,44 +262,46 @@ def update_code_package( :param storage_config: Configuration of storage buckets. """ - def add_function( + def add_benchmark( self, deployment_name: str, language_name: str, - code_package: "Benchmark", - function: "Function", + code_package: "CodePackage", + benchmark: "Benchmark", ): - if self.ignore_functions: + if self.ignore_benchmarks: return with self._lock: - benchmark_dir = os.path.join(self.cache_dir, code_package.benchmark) + benchmark_dir = os.path.join(self.cache_dir, code_package.name) language = code_package.language_name cache_config = os.path.join(benchmark_dir, "config.json") if os.path.exists(cache_config): - functions_config: Dict[str, Any] = {function.name: {**function.serialize()}} + benchmarks_config: Dict[str, Any] = {benchmark.name: {**benchmark.serialize()}} with open(cache_config, "r") as fp: cached_config = json.load(fp) - if "functions" not in cached_config[deployment_name][language]: - cached_config[deployment_name][language]["functions"] = functions_config + if "benchmarks" not in cached_config[deployment_name][language]: + cached_config[deployment_name][language]["benchmarks"] = benchmarks_config else: - cached_config[deployment_name][language]["functions"].update( - functions_config + cached_config[deployment_name][language]["benchmarks"].update( + benchmarks_config ) config = cached_config with open(cache_config, "w") as fp: json.dump(config, fp, indent=2) else: raise RuntimeError( - "Can't cache function {} for a non-existing code package!".format(function.name) + "Can't cache benchmark {} for a non-existing code package!".format( + benchmark.name + ) ) - def update_function(self, function: "Function"): - if self.ignore_functions: + def update_benchmark(self, benchmark: "Benchmark"): + if self.ignore_benchmarks: return with self._lock: - benchmark_dir = os.path.join(self.cache_dir, function.benchmark) + benchmark_dir = os.path.join(self.cache_dir, benchmark.code_package) cache_config = os.path.join(benchmark_dir, "config.json") if os.path.exists(cache_config): @@ -306,16 +310,18 @@ def update_function(self, function: "Function"): cached_config = json.load(fp) for deployment, cfg in cached_config.items(): for language, cfg2 in cfg.items(): - if "functions" not in cfg2: + if "benchmarks" not in cfg2: continue - for name, func in cfg2["functions"].items(): - if name == function.name: - cached_config[deployment][language]["functions"][ + for name, func in cfg2["benchmarks"].items(): + if name == benchmark.name: + cached_config[deployment][language]["benchmarks"][ name - ] = function.serialize() + ] = benchmark.serialize() with open(cache_config, "w") as fp: json.dump(cached_config, fp, indent=2) else: raise RuntimeError( - "Can't cache function {} for a non-existing code package!".format(function.name) + "Can't cache benchmark {} for a non-existing code package!".format( + benchmark.name + ) ) diff --git a/sebs/benchmark.py b/sebs/code_package.py similarity index 81% rename from sebs/benchmark.py rename to sebs/code_package.py index a631f2d8..e349a610 100644 --- a/sebs/benchmark.py +++ b/sebs/code_package.py @@ -10,7 +10,7 @@ from sebs.config import SeBSConfig from sebs.cache import Cache -from sebs.utils import find_benchmark, project_absolute_path, LoggingBase +from sebs.utils import find_package_code, project_absolute_path, LoggingBase from sebs.faas.storage import PersistentStorage from typing import TYPE_CHECKING @@ -19,7 +19,7 @@ from sebs.experiments.config import Language -class BenchmarkConfig: +class CodePackageConfig: def __init__(self, timeout: int, memory: int, languages: List["Language"]): self._timeout = timeout self._memory = memory @@ -39,10 +39,10 @@ def languages(self) -> List["Language"]: # FIXME: 3.7+ python with future annotations @staticmethod - def deserialize(json_object: dict) -> "BenchmarkConfig": + def deserialize(json_object: dict) -> "CodePackageConfig": from sebs.experiments.config import Language - return BenchmarkConfig( + return CodePackageConfig( json_object["timeout"], json_object["memory"], [Language.deserialize(x) for x in json_object["languages"]], @@ -62,35 +62,35 @@ def deserialize(json_object: dict) -> "BenchmarkConfig": """ -class Benchmark(LoggingBase): +class CodePackage(LoggingBase): @staticmethod def typename() -> str: - return "Benchmark" + return "CodePackage" @property - def benchmark(self): - return self._benchmark + def name(self): + return self._name @property - def benchmark_path(self): - return self._benchmark_path + def path(self): + return self._path @property - def benchmark_config(self) -> BenchmarkConfig: - return self._benchmark_config + def config(self) -> CodePackageConfig: + return self._config @property - def code_package(self) -> dict: - return self._code_package + def payload(self) -> dict: + return self._payload @property - def functions(self) -> Dict[str, Any]: - return self._functions + def benchmarks(self) -> Dict[str, Any]: + return self._benchmarks @property def code_location(self): - if self.code_package: - return os.path.join(self._cache_client.cache_dir, self.code_package["location"]) + if self.payload: + return os.path.join(self._cache_client.cache_dir, self.payload["location"]) else: return self._code_location @@ -128,8 +128,10 @@ def language_version(self): @property # noqa: A003 def hash(self): - path = os.path.join(self.benchmark_path, self.language_name) - self._hash_value = Benchmark.hash_directory(path, self._deployment_name, self.language_name) + path = os.path.join(self.path, self.language_name) + self._hash_value = CodePackage.hash_directory( + path, self._deployment_name, self.language_name + ) return self._hash_value @hash.setter # noqa: A003 @@ -141,7 +143,7 @@ def hash(self, val: str): def __init__( self, - benchmark: str, + name: str, deployment_name: str, config: "ExperimentConfig", system_config: SeBSConfig, @@ -150,27 +152,26 @@ def __init__( docker_client: docker.client, ): super().__init__() - self._benchmark = benchmark + self._name = name self._deployment_name = deployment_name self._experiment_config = config self._language = config.runtime.language self._language_version = config.runtime.version - self._benchmark_path = find_benchmark(self.benchmark, "benchmarks") - if not self._benchmark_path: - raise RuntimeError("Benchmark {benchmark} not found!".format(benchmark=self._benchmark)) - with open(os.path.join(self.benchmark_path, "config.json")) as json_file: - self._benchmark_config: BenchmarkConfig = BenchmarkConfig.deserialize( - json.load(json_file) - ) - if self.language not in self.benchmark_config.languages: + self._path = find_package_code(self.name, "benchmarks") + if not self._path: + raise RuntimeError("Benchmark {name} not found!".format(name=self._name)) + with open(os.path.join(self.path, "config.json")) as json_file: + self._config: CodePackageConfig = CodePackageConfig.deserialize(json.load(json_file)) + + if self.language not in self.config.languages: raise RuntimeError( - "Benchmark {} not available for language {}".format(self.benchmark, self.language) + "Benchmark {} not available for language {}".format(self.name, self.language) ) self._cache_client = cache_client self._docker_client = docker_client self._system_config = system_config self._hash_value = None - self._output_dir = os.path.join(output_dir, f"{benchmark}_code") + self._output_dir = os.path.join(output_dir, f"{name}_code") # verify existence of function in cache self.query_cache() @@ -197,6 +198,13 @@ def hash_directory(directory: str, deployment: str, language: str): path = os.path.join(directory, f) with open(path, "rb") as opened_file: hash_sum.update(opened_file.read()) + + # workflow definition + definition_path = os.path.join(directory, os.path.pardir, "definition.json") + if os.path.exists(definition_path): + with open(definition_path, "rb") as opened_file: + hash_sum.update(opened_file.read()) + # wrappers wrappers = project_absolute_path( "benchmarks", "wrappers", deployment, language, WRAPPERS[language] @@ -211,55 +219,63 @@ def serialize(self) -> dict: return {"size": self.code_size, "hash": self.hash} def query_cache(self): - self._code_package = self._cache_client.get_code_package( + self._payload = self._cache_client.get_code_package( deployment=self._deployment_name, - benchmark=self._benchmark, + benchmark=self._name, language=self.language_name, ) - self._functions = self._cache_client.get_functions( + self._benchmarks = self._cache_client.get_benchmarks( deployment=self._deployment_name, - benchmark=self._benchmark, + benchmark=self._name, language=self.language_name, ) - if self._code_package is not None: + if self._payload is not None: # compare hashes current_hash = self.hash - old_hash = self._code_package["hash"] - self._code_size = self._code_package["size"] + old_hash = self._payload["hash"] + self._code_size = self._payload["size"] self._is_cached = True self._is_cached_valid = current_hash == old_hash else: self._is_cached = False self._is_cached_valid = False - def copy_code(self, output_dir): + def get_code_files(self, include_config=True): FILES = { - "python": ["*.py", "requirements.txt*"], - "nodejs": ["*.js", "package.json"], + "python": ["*.py"], + "nodejs": ["*.js"], } - path = os.path.join(self.benchmark_path, self.language_name) + if include_config: + FILES["python"] += ["requirements.txt*", "*.json"] + FILES["nodejs"] += ["package.json", "*.json"] + + path = os.path.join(self.path, self.language_name) for file_type in FILES[self.language_name]: for f in glob.glob(os.path.join(path, file_type)): - shutil.copy2(os.path.join(path, f), output_dir) + yield os.path.join(path, f) + + def copy_code(self, output_dir: str): + for path in self.get_code_files(): + shutil.copy2(path, output_dir) def add_benchmark_data(self, output_dir): - cmd = "/bin/bash {benchmark_path}/init.sh {output_dir} false" + cmd = "/bin/bash {path}/init.sh {output_dir} false" paths = [ - self.benchmark_path, - os.path.join(self.benchmark_path, self.language_name), + self.path, + os.path.join(self.path, self.language_name), ] for path in paths: if os.path.exists(os.path.join(path, "init.sh")): out = subprocess.run( - cmd.format(benchmark_path=path, output_dir=output_dir), + cmd.format(path=path, output_dir=output_dir), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) self.logging.debug(out.stdout.decode("utf-8")) - def add_deployment_files(self, output_dir): + def add_deployment_files(self, output_dir: str, is_workflow: bool): handlers_dir = project_absolute_path( "benchmarks", "wrappers", self._deployment_name, self.language_name ) @@ -272,6 +288,17 @@ def add_deployment_files(self, output_dir): for file in handlers: shutil.copy2(file, os.path.join(output_dir)) + if self.language_name == "python": + handler_path = os.path.join(output_dir, "handler.py") + handler_function_path = os.path.join(output_dir, "handler_function.py") + handler_workflow_path = os.path.join(output_dir, "handler_workflow.py") + if is_workflow: + os.rename(handler_workflow_path, handler_path) + os.remove(handler_function_path) + else: + os.rename(handler_function_path, handler_path) + os.remove(handler_workflow_path) + def add_deployment_package_python(self, output_dir): # append to the end of requirements file packages = self._system_config.deployment_packages( @@ -279,8 +306,9 @@ def add_deployment_package_python(self, output_dir): ) if len(packages): with open(os.path.join(output_dir, "requirements.txt"), "a") as out: + out.write("\n") for package in packages: - out.write(package) + out.write(package + "\n") def add_deployment_package_nodejs(self, output_dir): # modify package.json @@ -349,7 +377,7 @@ def install_dependencies(self, output_dir): if not self._experiment_config.check_flag("docker_copy_build_files"): volumes = {os.path.abspath(output_dir): {"bind": "/mnt/function", "mode": "rw"}} package_script = os.path.abspath( - os.path.join(self._benchmark_path, self.language_name, "package.sh") + os.path.join(self._path, self.language_name, "package.sh") ) # does this benchmark has package.sh script? if os.path.exists(package_script): @@ -378,7 +406,7 @@ def install_dependencies(self, output_dir): stdout = self._docker_client.containers.run( "{}:{}".format(repo_name, image_name), volumes=volumes, - environment={"APP": self.benchmark}, + environment={"APP": self.name}, # user="1000:1000", user=uid, remove=True, @@ -390,7 +418,7 @@ def install_dependencies(self, output_dir): else: container = self._docker_client.containers.run( "{}:{}".format(repo_name, image_name), - environment={"APP": self.benchmark}, + environment={"APP": self.name}, # user="1000:1000", user=uid, # remove=True, @@ -443,17 +471,19 @@ def install_dependencies(self, output_dir): raise e def recalculate_code_size(self): - self._code_size = Benchmark.directory_size(self._output_dir) + self._code_size = CodePackage.directory_size(self._output_dir) return self._code_size def build( - self, deployment_build_step: Callable[[str, str, str], Tuple[str, int]] + self, + deployment_build_step: Callable[["CodePackage", str, bool], Tuple[str, int]], + is_workflow: bool, ) -> Tuple[bool, str]: # Skip build if files are up to date and user didn't enforce rebuild if self.is_cached and self.is_cached_valid: self.logging.info( - "Using cached benchmark {} at {}".format(self.benchmark, self.code_location) + "Using cached benchmark {} at {}".format(self.name, self.code_location) ) return False, self.code_location @@ -462,9 +492,9 @@ def build( if not self.is_cached else "cached code package is not up to date/build enforced." ) - self.logging.info("Building benchmark {}. Reason: {}".format(self.benchmark, msg)) + self.logging.info("Building benchmark {}. Reason: {}".format(self.name, msg)) # clear existing cache information - self._code_package = None + self._payload = None # create directory to be deployed if os.path.exists(self._output_dir): @@ -473,11 +503,11 @@ def build( self.copy_code(self._output_dir) self.add_benchmark_data(self._output_dir) - self.add_deployment_files(self._output_dir) + self.add_deployment_files(self._output_dir, is_workflow) self.add_deployment_package(self._output_dir) self.install_dependencies(self._output_dir) self._code_location, self._code_size = deployment_build_step( - os.path.abspath(self._output_dir), self.language_name, self.benchmark + self, os.path.abspath(self._output_dir), is_workflow ) self.logging.info( ( @@ -506,15 +536,15 @@ def build( :param client: Deployment client :param benchmark: - :param benchmark_path: + :param path: :param size: Benchmark workload size """ def prepare_input(self, storage: PersistentStorage, size: str): - benchmark_data_path = find_benchmark(self._benchmark, "benchmarks-data") - mod = load_benchmark_input(self._benchmark_path) + benchmark_data_path = find_package_code(self.name, "benchmarks-data") + mod = load_benchmark_input(self._path) buckets = mod.buckets_count() - storage.allocate_buckets(self.benchmark, buckets) + storage.allocate_buckets(self.name, buckets) # Get JSON and upload data as required by benchmark input_config = mod.generate_input( benchmark_data_path, @@ -532,9 +562,9 @@ def prepare_input(self, storage: PersistentStorage, size: str): def code_package_modify(self, filename: str, data: bytes): - if self.code_package_is_archive(): + if self.is_archive(): self._update_zip(self.code_location, filename, data) - new_size = self.code_package_recompute_size() / 1024.0 / 1024.0 + new_size = self.recompute_size() / 1024.0 / 1024.0 self.logging.info(f"Modified zip package {self.code_location}, new size {new_size} MB") else: raise NotImplementedError() @@ -544,13 +574,13 @@ def code_package_modify(self, filename: str, data: bytes): Azure: directory """ - def code_package_is_archive(self) -> bool: + def is_archive(self) -> bool: if os.path.isfile(self.code_location): extension = os.path.splitext(self.code_location)[1] return extension in [".zip"] return False - def code_package_recompute_size(self) -> float: + def recompute_size(self) -> float: bytes_size = os.path.getsize(self.code_location) self._code_size = bytes_size return bytes_size @@ -588,7 +618,7 @@ def _update_zip(zipname: str, filename: str, data: bytes): """ -class BenchmarkModuleInterface: +class CodePackageModuleInterface: @staticmethod def buckets_count() -> Tuple[int, int]: pass @@ -604,11 +634,11 @@ def generate_input( pass -def load_benchmark_input(benchmark_path: str) -> BenchmarkModuleInterface: +def load_benchmark_input(path: str) -> CodePackageModuleInterface: # Look for input generator file in the directory containing benchmark import importlib.machinery - loader = importlib.machinery.SourceFileLoader("input", os.path.join(benchmark_path, "input.py")) + loader = importlib.machinery.SourceFileLoader("input", os.path.join(path, "input.py")) spec = importlib.util.spec_from_loader(loader.name, loader) assert spec mod = importlib.util.module_from_spec(spec) diff --git a/sebs/experiments/eviction_model.py b/sebs/experiments/eviction_model.py index 8524c5a4..4d55c66c 100644 --- a/sebs/experiments/eviction_model.py +++ b/sebs/experiments/eviction_model.py @@ -7,7 +7,7 @@ from multiprocessing.pool import AsyncResult, ThreadPool from sebs.faas.system import System as FaaSSystem -from sebs.faas.function import Function, Trigger +from sebs.faas.benchmark import Function, Trigger from sebs.experiments import Experiment, ExperimentResult from sebs.experiments.config import Config as ExperimentConfig from sebs.utils import serialize @@ -183,7 +183,7 @@ def prepare(self, sebs_client: "SeBS", deployment_client: FaaSSystem): ) self._deployment_client = deployment_client self._result = ExperimentResult(self.config, deployment_client.config) - name = deployment_client.default_function_name(self._benchmark) + name = deployment_client.default_benchmark_name(self._benchmark) self.functions_names = [ f"{name}-{time}-{copy}" for time in self.times diff --git a/sebs/experiments/invocation_overhead.py b/sebs/experiments/invocation_overhead.py index 76f9a41a..11bbe403 100644 --- a/sebs/experiments/invocation_overhead.py +++ b/sebs/experiments/invocation_overhead.py @@ -5,7 +5,7 @@ from datetime import datetime from typing import Dict, TYPE_CHECKING -from sebs.benchmark import Benchmark +from sebs.code_package import CodePackage from sebs.faas.system import System as FaaSSystem from sebs.experiments.experiment import Experiment from sebs.experiments.config import Config as ExperimentConfig @@ -15,7 +15,7 @@ class CodePackageSize: - def __init__(self, deployment_client: FaaSSystem, benchmark: Benchmark, settings: dict): + def __init__(self, deployment_client: FaaSSystem, benchmark: CodePackage, settings: dict): import math from numpy import linspace @@ -24,9 +24,9 @@ def __init__(self, deployment_client: FaaSSystem, benchmark: Benchmark, settings settings["code_package_end"], settings["code_package_points"], ) - from sebs.utils import find_benchmark + from sebs.utils import find_package_code - self._benchmark_path = find_benchmark("030.clock-synchronization", "benchmarks") + self._benchmark_path = find_package_code("030.clock-synchronization", "benchmarks") self._benchmark = benchmark random.seed(1410) @@ -73,7 +73,7 @@ def prepare(self, sebs_client: "SeBS", deployment_client: FaaSSystem): # deploy network test function from sebs import SeBS # noqa - from sebs.faas.function import Trigger + from sebs.faas.benchmark import Trigger self._benchmark = sebs_client.get_benchmark( "030.clock-synchronization", deployment_client, self.config diff --git a/sebs/experiments/network_ping_pong.py b/sebs/experiments/network_ping_pong.py index 303f6f53..a95506de 100644 --- a/sebs/experiments/network_ping_pong.py +++ b/sebs/experiments/network_ping_pong.py @@ -10,7 +10,7 @@ from multiprocessing.dummy import Pool as ThreadPool from sebs.faas.system import System as FaaSSystem -from sebs.faas.function import Trigger +from sebs.faas.benchmark import Trigger from sebs.experiments.experiment import Experiment from sebs.experiments.config import Config as ExperimentConfig diff --git a/sebs/experiments/perf_cost.py b/sebs/experiments/perf_cost.py index 3fc81482..80514db4 100644 --- a/sebs/experiments/perf_cost.py +++ b/sebs/experiments/perf_cost.py @@ -6,7 +6,7 @@ from typing import List, TYPE_CHECKING from sebs.faas.system import System as FaaSSystem -from sebs.faas.function import Trigger +from sebs.faas.benchmark import Trigger from sebs.experiments.experiment import Experiment from sebs.experiments.result import Result as ExperimentResult from sebs.experiments.config import Config as ExperimentConfig @@ -354,7 +354,10 @@ def process( ) as out_f: out_f.write( serialize( - {**json.loads(serialize(experiments)), "statistics": statistics} + { + **json.loads(serialize(experiments)), + "statistics": statistics, + } ) ) for func in experiments.functions(): diff --git a/sebs/experiments/result.py b/sebs/experiments/result.py index 1a56684c..5087b904 100644 --- a/sebs/experiments/result.py +++ b/sebs/experiments/result.py @@ -3,7 +3,7 @@ from sebs.cache import Cache from sebs.faas.config import Config as DeploymentConfig -from sebs.faas.function import Function, ExecutionResult +from sebs.faas.benchmark import Function, ExecutionResult from sebs.utils import LoggingHandlers from sebs.experiments.config import Config as ExperimentConfig diff --git a/sebs/faas/function.py b/sebs/faas/benchmark.py similarity index 86% rename from sebs/faas/function.py rename to sebs/faas/benchmark.py index 56688779..a96e0cd2 100644 --- a/sebs/faas/function.py +++ b/sebs/faas/benchmark.py @@ -6,6 +6,8 @@ from enum import Enum from typing import Callable, Dict, List, Optional # noqa +from google.cloud.workflows.executions_v1beta.types import Execution + from sebs.utils import LoggingBase """ @@ -147,6 +149,12 @@ def parse_benchmark_output(self, output: dict): / timedelta(microseconds=1) ) + def parse_benchmark_execution(self, execution: Execution): + self.output = json.loads(execution.result) + self.times.benchmark = int( + (execution.start_time - execution.end_time) / timedelta(microseconds=1) + ) + @staticmethod def deserialize(cached_config: dict) -> "ExecutionResult": ret = ExecutionResult() @@ -203,7 +211,9 @@ def _http_invoke(self, payload: dict, url: str) -> ExecutionResult: output = json.loads(data.getvalue()) if status_code != 200: - self.logging.error("Invocation on URL {} failed!".format(url)) + self.logging.error( + "Invocation on URL {} failed with status code {}!".format(url, status_code) + ) self.logging.error("Output: {}".format(output)) raise RuntimeError(f"Failed invocation of function! Output: {output}") @@ -216,7 +226,9 @@ def _http_invoke(self, payload: dict, url: str) -> ExecutionResult: result.parse_benchmark_output(output) return result except json.decoder.JSONDecodeError: - self.logging.error("Invocation on URL {} failed!".format(url)) + self.logging.error( + "Invocation on URL {} failed with status code {}!".format(url, status_code) + ) self.logging.error("Output: {}".format(data.getvalue().decode())) raise RuntimeError(f"Failed invocation of function! Output: {data.getvalue().decode()}") @@ -238,23 +250,23 @@ def async_invoke(self, payload: dict) -> concurrent.futures.Future: def serialize(self) -> dict: pass - @staticmethod + @classmethod @abstractmethod - def deserialize(cached_config: dict) -> "Trigger": + def deserialize(cls, cached_config: dict) -> "Trigger": pass """ - Abstraction base class for FaaS function. Contains a list of associated triggers + Abstraction base class for FaaS benchmarks. Contains a list of associated triggers and might implement non-trigger execution if supported by the SDK. Example: direct function invocation through AWS boto3 SDK. """ -class Function(LoggingBase): - def __init__(self, benchmark: str, name: str, code_hash: str): +class Benchmark(LoggingBase): + def __init__(self, code_package: str, name: str, code_hash: str): super().__init__() - self._benchmark = benchmark + self._code_package = code_package self._name = name self._code_package_hash = code_hash self._updated_code = False @@ -265,8 +277,8 @@ def name(self): return self._name @property - def benchmark(self): - return self._benchmark + def code_package(self): + return self._code_package @property def code_package_hash(self): @@ -303,13 +315,27 @@ def serialize(self) -> dict: return { "name": self._name, "hash": self._code_package_hash, - "benchmark": self._benchmark, + "code_package": self._code_package, "triggers": [ obj.serialize() for t_type, triggers in self._triggers.items() for obj in triggers ], } + @staticmethod + @abstractmethod + def deserialize(cached_config: dict) -> "Benchmark": + pass + + +class Function(Benchmark): @staticmethod @abstractmethod def deserialize(cached_config: dict) -> "Function": pass + + +class Workflow(Benchmark): + @staticmethod + @abstractmethod + def deserialize(cached_config: dict) -> "Workflow": + pass diff --git a/sebs/faas/fsm.py b/sebs/faas/fsm.py new file mode 100644 index 00000000..8e59be83 --- /dev/null +++ b/sebs/faas/fsm.py @@ -0,0 +1,167 @@ +from abc import ABC +from abc import abstractmethod +from typing import Optional, List, Callable, Union, Dict, Type, Tuple +import json + + +class State(ABC): + def __init__(self, name: str): + self.name = name + + @staticmethod + def deserialize(name: str, payload: dict) -> "State": + cls = _STATE_TYPES[payload["type"]] + return cls.deserialize(name, payload) + + +class Task(State): + def __init__(self, name: str, func_name: str, next: Optional[str]): + self.name = name + self.func_name = func_name + self.next = next + + @classmethod + def deserialize(cls, name: str, payload: dict) -> "Task": + return cls(name=name, func_name=payload["func_name"], next=payload.get("next")) + + +class Switch(State): + class Case: + def __init__(self, var: str, op: str, val: str, next: str): + self.var = var + self.op = op + self.val = val + self.next = next + + @staticmethod + def deserialize(payload: dict) -> "Switch.Case": + return Switch.Case(**payload) + + def __init__(self, name: str, cases: List[Case], default: Optional[str]): + self.name = name + self.cases = cases + self.default = default + + @classmethod + def deserialize(cls, name: str, payload: dict) -> "Switch": + cases = [Switch.Case.deserialize(c) for c in payload["cases"]] + + return cls(name=name, cases=cases, default=payload["default"]) + + +class Map(State): + def __init__(self, name: str, func_name: str, array: str, next: Optional[str]): + self.name = name + self.func_name = func_name + self.array = array + self.next = next + + @classmethod + def deserialize(cls, name: str, payload: dict) -> "Map": + return cls( + name=name, + func_name=payload["func_name"], + array=payload["array"], + next=payload.get("next"), + ) + + +class Repeat(State): + def __init__(self, name: str, func_name: str, count: int, next: Optional[str]): + self.name = name + self.func_name = func_name + self.count = count + self.next = next + + @classmethod + def deserialize(cls, name: str, payload: dict) -> "Repeat": + return cls(name=name, func_name=payload["func_name"], count=payload["count"], next=payload.get("next")) + + +class Loop(State): + def __init__(self, name: str, func_name: str, array: str, next: Optional[str]): + self.name = name + self.func_name = func_name + self.array = array + self.next = next + + @classmethod + def deserialize(cls, name: str, payload: dict) -> "Loop": + return cls( + name=name, + func_name=payload["func_name"], + array=payload["array"], + next=payload.get("next"), + ) + + +_STATE_TYPES: Dict[str, Type[State]] = {"task": Task, "switch": Switch, "map": Map, "repeat": Repeat, "loop": Loop} + + +class Generator(ABC): + def __init__(self, export_func: Callable[[dict], str] = json.dumps): + self._export_func = export_func + + def parse(self, path: str): + with open(path) as f: + definition = json.load(f) + + self.states = {n: State.deserialize(n, s) for n, s in definition["states"].items()} + self.root = self.states[definition["root"]] + + def generate(self) -> str: + states = list(self.states.values()) + payloads = [] + for s in states: + obj = self.encode_state(s) + if isinstance(obj, dict): + payloads.append(obj) + elif isinstance(obj, list): + payloads += obj + else: + raise ValueError("Unknown encoded state returned.") + + definition = self.postprocess(payloads) + + return self._export_func(definition) + + def postprocess(self, payloads: List[dict]) -> dict: + pass + + def encode_state(self, state: State) -> Union[dict, List[dict]]: + if isinstance(state, Task): + return self.encode_task(state) + elif isinstance(state, Switch): + return self.encode_switch(state) + elif isinstance(state, Map): + return self.encode_map(state) + elif isinstance(state, Repeat): + return self.encode_repeat(state) + else: + raise ValueError(f"Unknown state of type {type(state)}.") + + @abstractmethod + def encode_task(self, state: Task) -> Union[dict, List[dict]]: + pass + + @abstractmethod + def encode_switch(self, state: Switch) -> Union[dict, List[dict]]: + pass + + @abstractmethod + def encode_map(self, state: Map) -> Union[dict, List[dict]]: + pass + + def encode_repeat(self, state: Repeat) -> Union[dict, List[dict]]: + tasks = [] + for i in range(state.count): + name = state.name if i == 0 else f"{state.name}_{i}" + next = state.next if i == state.count-1 else f"{state.name}_{i+1}" + task = Task(name, state.func_name, next) + tasks.append(self.encode_task(task)) + + return tasks + + @abstractmethod + def encode_loop(self, state: Loop) -> Union[dict, List[dict]]: + pass diff --git a/sebs/faas/system.py b/sebs/faas/system.py index cdc3a656..0bfc410a 100644 --- a/sebs/faas/system.py +++ b/sebs/faas/system.py @@ -5,10 +5,10 @@ import docker -from sebs.benchmark import Benchmark +from sebs.code_package import CodePackage from sebs.cache import Cache from sebs.config import SeBSConfig -from sebs.faas.function import Function, Trigger, ExecutionResult +from sebs.faas.benchmark import Benchmark, Function, Trigger, ExecutionResult, Workflow from sebs.faas.storage import PersistentStorage from sebs.utils import LoggingBase from .config import Config @@ -65,6 +65,11 @@ def config(self) -> Config: def function_type() -> "Type[Function]": pass + @staticmethod + @abstractmethod + def workflow_type() -> "Type[Workflow]": + pass + """ Initialize the system. After the call the local or remot FaaS system should be ready to allocate functions, manage @@ -104,19 +109,25 @@ def get_storage(self, replace_existing: bool) -> PersistentStorage: """ @abstractmethod - def package_code(self, directory: str, language_name: str, benchmark: str) -> Tuple[str, int]: + def package_code( + self, code_package: CodePackage, directory: str, is_workflow: bool + ) -> Tuple[str, int]: pass @abstractmethod - def create_function(self, code_package: Benchmark, func_name: str) -> Function: + def create_function(self, code_package: CodePackage, func_name: str) -> Function: pass @abstractmethod - def cached_function(self, function: Function): + def create_workflow(self, code_package: CodePackage, workflow_name: str) -> Workflow: pass @abstractmethod - def update_function(self, function: Function, code_package: Benchmark): + def cached_benchmark(self, benchmark: Benchmark): + pass + + @abstractmethod + def update_function(self, function: Function, code_package: CodePackage): pass """ @@ -132,8 +143,7 @@ def update_function(self, function: Function, code_package: Benchmark): """ - def get_function(self, code_package: Benchmark, func_name: Optional[str] = None) -> Function: - + def get_function(self, code_package: CodePackage, func_name: Optional[str] = None) -> Function: if code_package.language_version not in self.system_config.supported_language_versions( self.name(), code_package.language_name ): @@ -146,8 +156,8 @@ def get_function(self, code_package: Benchmark, func_name: Optional[str] = None) ) if not func_name: - func_name = self.default_function_name(code_package) - rebuilt, _ = code_package.build(self.package_code) + func_name = self.default_benchmark_name(code_package) + rebuilt, _ = code_package.build(self.package_code, False) """ There's no function with that name? @@ -156,8 +166,8 @@ def get_function(self, code_package: Benchmark, func_name: Optional[str] = None) b) no -> retrieve function from the cache. Function code in cloud will be updated if the local version is different. """ - functions = code_package.functions - if not functions or func_name not in functions: + benchmarks = code_package.benchmarks + if not benchmarks or func_name not in benchmarks: msg = ( "function name not provided." if not func_name @@ -165,20 +175,20 @@ def get_function(self, code_package: Benchmark, func_name: Optional[str] = None) ) self.logging.info("Creating new function! Reason: " + msg) function = self.create_function(code_package, func_name) - self.cache_client.add_function( + self.cache_client.add_benchmark( deployment_name=self.name(), language_name=code_package.language_name, code_package=code_package, - function=function, + benchmark=function, ) code_package.query_cache() return function else: # retrieve function - cached_function = functions[func_name] + cached_function = benchmarks[func_name] code_location = code_package.code_location function = self.function_type().deserialize(cached_function) - self.cached_function(function) + self.cached_benchmark(function) self.logging.info( "Using cached function {fname} in {loc}".format(fname=func_name, loc=code_location) ) @@ -193,21 +203,96 @@ def get_function(self, code_package: Benchmark, func_name: Optional[str] = None) self.update_function(function, code_package) function.code_package_hash = code_package.hash function.updated_code = True - self.cache_client.add_function( + self.cache_client.add_benchmark( deployment_name=self.name(), language_name=code_package.language_name, code_package=code_package, - function=function, + benchmark=function, ) code_package.query_cache() return function @abstractmethod - def default_function_name(self, code_package: Benchmark) -> str: + def update_workflow(self, workflow: Workflow, code_package: CodePackage): + pass + + def get_workflow(self, code_package: CodePackage, workflow_name: Optional[str] = None): + if code_package.language_version not in self.system_config.supported_language_versions( + self.name(), code_package.language_name + ): + raise Exception( + "Unsupported {language} version {version} in {system}!".format( + language=code_package.language_name, + version=code_package.language_version, + system=self.name(), + ) + ) + + if not workflow_name: + workflow_name = self.default_benchmark_name(code_package) + rebuilt, _ = code_package.build(self.package_code, True) + + """ + There's no function with that name? + a) yes -> create new function. Implementation might check if a function + with that name already exists in the cloud and update its code. + b) no -> retrieve function from the cache. Function code in cloud will + be updated if the local version is different. + """ + benchmarks = code_package.benchmarks + if not benchmarks or workflow_name not in benchmarks: + msg = ( + "workflow name not provided." + if not workflow_name + else "workflow {} not found in cache.".format(workflow_name) + ) + self.logging.info("Creating new workflow! Reason: " + msg) + workflow = self.create_workflow(code_package, workflow_name) + self.cache_client.add_benchmark( + deployment_name=self.name(), + language_name=code_package.language_name, + code_package=code_package, + benchmark=workflow, + ) + code_package.query_cache() + return workflow + else: + # retrieve function + cached_workflow = benchmarks[workflow_name] + code_location = code_package.code_location + workflow = self.workflow_type().deserialize(cached_workflow) + self.cached_benchmark(workflow) + self.logging.info( + "Using cached workflow {workflow_name} in {loc}".format( + workflow_name=workflow_name, loc=code_location + ) + ) + # is the function up-to-date? + if workflow.code_package_hash != code_package.hash or rebuilt: + self.logging.info( + f"Cached workflow {workflow_name} with hash " + f"{workflow.code_package_hash} is not up to date with " + f"current build {code_package.hash} in " + f"{code_location}, updating cloud version!" + ) + self.update_workflow(workflow, code_package) + workflow.code_package_hash = code_package.hash + workflow.updated_code = True + self.cache_client.add_benchmark( + deployment_name=self.name(), + language_name=code_package.language_name, + code_package=code_package, + benchmark=workflow, + ) + code_package.query_cache() + return workflow + + @abstractmethod + def default_benchmark_name(self, code_package: CodePackage) -> str: pass @abstractmethod - def enforce_cold_start(self, functions: List[Function], code_package: Benchmark): + def enforce_cold_start(self, functions: List[Function], code_package: CodePackage): pass @abstractmethod @@ -221,8 +306,24 @@ def download_metrics( ): pass + def create_trigger(self, obj, trigger_type: Trigger.TriggerType) -> Trigger: + if isinstance(obj, Function): + return self.create_function_trigger(obj, trigger_type) + elif isinstance(obj, Workflow): + return self.create_workflow_trigger(obj, trigger_type) + else: + raise TypeError("Cannot create trigger for {obj}") + + @abstractmethod + def create_function_trigger( + self, function: Function, trigger_type: Trigger.TriggerType + ) -> Trigger: + pass + @abstractmethod - def create_trigger(self, function: Function, trigger_type: Trigger.TriggerType) -> Trigger: + def create_workflow_trigger( + self, workflow: Workflow, trigger_type: Trigger.TriggerType + ) -> Trigger: pass # @abstractmethod diff --git a/sebs/gcp/config.py b/sebs/gcp/config.py index c4624ad3..52f234ba 100644 --- a/sebs/gcp/config.py +++ b/sebs/gcp/config.py @@ -144,10 +144,11 @@ class GCPConfig(Config): _project_name: str - def __init__(self, credentials: GCPCredentials, resources: GCPResources): + def __init__(self, credentials: GCPCredentials, resources: GCPResources, redis_host: str): super().__init__() self._credentials = credentials self._resources = resources + self._redis_host = redis_host @property def region(self) -> str: @@ -165,12 +166,16 @@ def credentials(self) -> GCPCredentials: def resources(self) -> GCPResources: return self._resources + @property + def redis_host(self) -> str: + return self._redis_host + @staticmethod def deserialize(config: dict, cache: Cache, handlers: LoggingHandlers) -> "Config": cached_config = cache.get_config("gcp") credentials = cast(GCPCredentials, GCPCredentials.deserialize(config, cache, handlers)) resources = cast(GCPResources, GCPResources.deserialize(config, cache, handlers)) - config_obj = GCPConfig(credentials, resources) + config_obj = GCPConfig(credentials, resources, cached_config["redis_host"]) config_obj.logging_handlers = handlers if cached_config: config_obj.logging.info("Loading cached config for GCP") @@ -182,7 +187,8 @@ def deserialize(config: dict, cache: Cache, handlers: LoggingHandlers) -> "Confi if "project_name" not in config or not config["project_name"]: if "GCP_PROJECT_NAME" in os.environ: GCPConfig.initialize( - config_obj, {**config, "project_name": os.environ["GCP_PROJECT_NAME"]} + config_obj, + {**config, "project_name": os.environ["GCP_PROJECT_NAME"]}, ) else: raise RuntimeError( @@ -219,6 +225,7 @@ def initialize(cfg: Config, dct: dict): config = cast(GCPConfig, cfg) config._project_name = dct["project_name"] config._region = dct["region"] + config._redis_host = dct["redis_host"] def serialize(self) -> dict: out = { @@ -227,6 +234,7 @@ def serialize(self) -> dict: "region": self._region, "credentials": self._credentials.serialize(), "resources": self._resources.serialize(), + "redis_host": self._redis_host, } return out diff --git a/sebs/gcp/function.py b/sebs/gcp/function.py index 80d32096..317781cf 100644 --- a/sebs/gcp/function.py +++ b/sebs/gcp/function.py @@ -1,6 +1,6 @@ from typing import cast, Optional -from sebs.faas.function import Function +from sebs.faas.benchmark import Function from sebs.gcp.storage import GCPStorage @@ -33,12 +33,12 @@ def serialize(self) -> dict: @staticmethod def deserialize(cached_config: dict) -> "GCPFunction": - from sebs.faas.function import Trigger - from sebs.gcp.triggers import LibraryTrigger, HTTPTrigger + from sebs.faas.benchmark import Trigger + from sebs.gcp.triggers import FunctionLibraryTrigger, HTTPTrigger ret = GCPFunction( cached_config["name"], - cached_config["benchmark"], + cached_config["code_package"], cached_config["hash"], cached_config["timeout"], cached_config["memory"], @@ -47,7 +47,7 @@ def deserialize(cached_config: dict) -> "GCPFunction": for trigger in cached_config["triggers"]: trigger_type = cast( Trigger, - {"Library": LibraryTrigger, "HTTP": HTTPTrigger}.get(trigger["type"]), + {"Library": FunctionLibraryTrigger, "HTTP": HTTPTrigger}.get(trigger["type"]), ) assert trigger_type, "Unknown trigger type {}".format(trigger["type"]) ret.add_trigger(trigger_type.deserialize(trigger)) diff --git a/sebs/gcp/gcp.py b/sebs/gcp/gcp.py index 230a8339..09906f34 100644 --- a/sebs/gcp/gcp.py +++ b/sebs/gcp/gcp.py @@ -11,18 +11,20 @@ from googleapiclient.discovery import build from googleapiclient.errors import HttpError -from google.cloud import monitoring_v3 +from google.cloud import monitoring_v3 # type: ignore from sebs.cache import Cache from sebs.config import SeBSConfig -from sebs.benchmark import Benchmark -from ..faas.function import Function, Trigger +from sebs.code_package import CodePackage +from sebs.faas.benchmark import Benchmark, Function, Trigger, Workflow from .storage import PersistentStorage from ..faas.system import System from sebs.gcp.config import GCPConfig from sebs.gcp.storage import GCPStorage from sebs.gcp.function import GCPFunction -from sebs.utils import LoggingHandlers +from sebs.gcp.workflow import GCPWorkflow +from sebs.gcp.generator import GCPGenerator +from sebs.utils import LoggingHandlers, replace_string_in_file """ This class provides basic abstractions for the FaaS system. @@ -63,6 +65,10 @@ def typename(): def function_type() -> "Type[Function]": return GCPFunction + @staticmethod + def workflow_type() -> "Type[Workflow]": + return GCPWorkflow + """ Initialize the system. After the call the local or remote FaaS system should be ready to allocate functions, manage @@ -73,11 +79,15 @@ def function_type() -> "Type[Function]": def initialize(self, config: Dict[str, str] = {}): self.function_client = build("cloudfunctions", "v1", cache_discovery=False) + self.workflow_client = build("workflows", "v1", cache_discovery=False) self.get_storage() def get_function_client(self): return self.function_client + def get_workflow_client(self): + return self.workflow_client + """ Access persistent storage instance. It might be a remote and truly persistent service (AWS S3, Azure Blob..), @@ -100,12 +110,12 @@ def get_storage( return self.storage @staticmethod - def default_function_name(code_package: Benchmark) -> str: + def default_benchmark_name(code_package: CodePackage) -> str: # Create function name func_name = "{}-{}-{}".format( - code_package.benchmark, + code_package.name, code_package.language_name, - code_package.benchmark_config.memory, + code_package.config.memory, ) return GCP.format_function_name(func_name) @@ -131,8 +141,9 @@ def format_function_name(func_name: str) -> str: :return: path to packaged code and its size """ - def package_code(self, directory: str, language_name: str, benchmark: str) -> Tuple[str, int]: - + def package_code( + self, code_package: CodePackage, directory: str, is_workflow: bool + ) -> Tuple[str, int]: CONFIG_FILES = { "python": ["handler.py", ".python_packages"], "nodejs": ["handler.js", "node_modules"], @@ -141,7 +152,8 @@ def package_code(self, directory: str, language_name: str, benchmark: str) -> Tu "python": ("handler.py", "main.py"), "nodejs": ("handler.js", "index.js"), } - package_config = CONFIG_FILES[language_name] + package_config = CONFIG_FILES[code_package.language_name] + function_dir = os.path.join(directory, "function") os.makedirs(function_dir) for file in os.listdir(directory): @@ -149,16 +161,14 @@ def package_code(self, directory: str, language_name: str, benchmark: str) -> Tu file = os.path.join(directory, file) shutil.move(file, function_dir) - requirements = open(os.path.join(directory, "requirements.txt"), "w") - requirements.write("google-cloud-storage") - requirements.close() - # rename handler function.py since in gcp it has to be caled main.py - old_name, new_name = HANDLER[language_name] + old_name, new_name = HANDLER[code_package.language_name] old_path = os.path.join(directory, old_name) new_path = os.path.join(directory, new_name) shutil.move(old_path, new_path) + replace_string_in_file(new_path, "{{REDIS_HOST}}", f'"{self.config.redis_host}"') + """ zip the whole directroy (the zip-file gets uploaded to gcp later) @@ -170,7 +180,7 @@ def package_code(self, directory: str, language_name: str, benchmark: str) -> Tu which leads to a "race condition" when running several benchmarks in parallel, since a change of the current directory is NOT Thread specfic. """ - benchmark_archive = "{}.zip".format(os.path.join(directory, benchmark)) + benchmark_archive = "{}.zip".format(os.path.join(directory, code_package.name)) GCP.recursive_zip(directory, benchmark_archive) logging.info("Created {} archive".format(benchmark_archive)) @@ -181,15 +191,15 @@ def package_code(self, directory: str, language_name: str, benchmark: str) -> Tu # rename the main.py back to handler.py shutil.move(new_path, old_path) - return os.path.join(directory, "{}.zip".format(benchmark)), bytes_size + return os.path.join(directory, "{}.zip".format(code_package.name)), bytes_size - def create_function(self, code_package: Benchmark, func_name: str) -> "GCPFunction": + def create_function(self, code_package: CodePackage, func_name: str) -> "GCPFunction": package = code_package.code_location - benchmark = code_package.benchmark + benchmark = code_package.name language_runtime = code_package.language_version - timeout = code_package.benchmark_config.timeout - memory = code_package.benchmark_config.memory + timeout = code_package.config.timeout + memory = code_package.config.memory code_bucket: Optional[str] = None storage_client = self.get_storage() location = self.config.region @@ -202,7 +212,6 @@ def create_function(self, code_package: Benchmark, func_name: str) -> "GCPFuncti full_func_name = GCP.get_full_function_name(project_name, location, func_name) get_req = self.function_client.projects().locations().functions().get(name=full_func_name) - try: get_req.execute() except HttpError: @@ -211,9 +220,7 @@ def create_function(self, code_package: Benchmark, func_name: str) -> "GCPFuncti .locations() .functions() .create( - location="projects/{project_name}/locations/{location}".format( - project_name=project_name, location=location - ), + location=GCP.get_location(project_name, location), body={ "name": full_func_name, "entryPoint": "handler", @@ -237,7 +244,10 @@ def create_function(self, code_package: Benchmark, func_name: str) -> "GCPFuncti body={ "policy": { "bindings": [ - {"role": "roles/cloudfunctions.invoker", "members": ["allUsers"]} + { + "role": "roles/cloudfunctions.invoker", + "members": ["allUsers"], + } ] } }, @@ -263,19 +273,20 @@ def create_function(self, code_package: Benchmark, func_name: str) -> "GCPFuncti ) self.update_function(function, code_package) - # Add LibraryTrigger to a new function - from sebs.gcp.triggers import LibraryTrigger + # Add LibraryFunctionTrigger to a new function + from sebs.gcp.triggers import FunctionLibraryTrigger - trigger = LibraryTrigger(func_name, self) + trigger = FunctionLibraryTrigger(func_name, self) trigger.logging_handlers = self.logging_handlers function.add_trigger(trigger) return function - def create_trigger(self, function: Function, trigger_type: Trigger.TriggerType) -> Trigger: - from sebs.gcp.triggers import HTTPTrigger - + def create_function_trigger( + self, function: Function, trigger_type: Trigger.TriggerType + ) -> Trigger: if trigger_type == Trigger.TriggerType.HTTP: + from sebs.gcp.triggers import HTTPTrigger location = self.config.region project_name = self.config.project_name @@ -284,6 +295,7 @@ def create_trigger(self, function: Function, trigger_type: Trigger.TriggerType) our_function_req = ( self.function_client.projects().locations().functions().get(name=full_func_name) ) + deployed = False while not deployed: status_res = our_function_req.execute() @@ -300,27 +312,27 @@ def create_trigger(self, function: Function, trigger_type: Trigger.TriggerType) trigger.logging_handlers = self.logging_handlers function.add_trigger(trigger) - self.cache_client.update_function(function) + self.cache_client.update_benchmark(function) return trigger - def cached_function(self, function: Function): + def cached_benchmark(self, benchmark: Benchmark): - from sebs.faas.function import Trigger + from sebs.faas.benchmark import Trigger from sebs.gcp.triggers import LibraryTrigger - for trigger in function.triggers(Trigger.TriggerType.LIBRARY): + for trigger in benchmark.triggers(Trigger.TriggerType.LIBRARY): gcp_trigger = cast(LibraryTrigger, trigger) gcp_trigger.logging_handlers = self.logging_handlers gcp_trigger.deployment_client = self - def update_function(self, function: Function, code_package: Benchmark): + def update_function(self, function: Function, code_package: CodePackage): function = cast(GCPFunction, function) language_runtime = code_package.language_version code_package_name = os.path.basename(code_package.code_location) storage = cast(GCPStorage, self.get_storage()) - bucket = function.code_bucket(code_package.benchmark, storage) + bucket = function.code_bucket(code_package.name, storage) storage.upload(bucket, code_package.code_location, code_package_name) self.logging.info(f"Uploaded new code package to {bucket}/{code_package_name}") full_func_name = GCP.get_full_function_name( @@ -356,6 +368,196 @@ def update_function(self, function: Function, code_package: Benchmark): def get_full_function_name(project_name: str, location: str, func_name: str): return f"projects/{project_name}/locations/{location}/functions/{func_name}" + def create_workflow(self, code_package: CodePackage, workflow_name: str) -> "GCPWorkflow": + from sebs.gcp.triggers import HTTPTrigger + + benchmark = code_package.name + timeout = code_package.config.timeout + memory = code_package.config.memory + code_bucket: Optional[str] = None + location = self.config.region + project_name = self.config.project_name + + # Make sure we have a valid workflow benchmark + definition_path = os.path.join(code_package.path, "definition.json") + if not os.path.exists(definition_path): + raise ValueError(f"No workflow definition found for {workflow_name}") + + # First we create a function for each code file + prefix = workflow_name + "___" + code_files = list(code_package.get_code_files(include_config=False)) + func_names = [os.path.splitext(os.path.basename(p))[0] for p in code_files] + funcs = [self.create_function(code_package, prefix + fn) for fn in func_names] + + # generate workflow definition.json + triggers = [self.create_function_trigger(f, Trigger.TriggerType.HTTP) for f in funcs] + urls = [cast(HTTPTrigger, t).url for t in triggers] + func_triggers = {n: u for (n, u) in zip(func_names, urls)} + + gen = GCPGenerator(workflow_name, func_triggers) + gen.parse(definition_path) + definition = gen.generate() + + # map functions require their own workflows + parent = GCP.get_location(project_name, location) + for map_id, map_def in gen.generate_maps(): + full_workflow_name = GCP.get_full_workflow_name(project_name, location, map_id) + create_req = ( + self.workflow_client.projects() # type: ignore + .locations() + .workflows() + .create( + parent=parent, + workflowId=map_id, + body={ + "name": full_workflow_name, + "sourceContents": map_def, + }, + ) + ) + create_req.execute() + self.logging.info(f"Map workflow {map_id} has been created!") + + full_workflow_name = GCP.get_full_workflow_name(project_name, location, workflow_name) + get_req = ( + self.workflow_client.projects() # type: ignore + .locations() + .workflows() + .get(name=full_workflow_name) + ) + + try: + get_req.execute() + except HttpError: + create_req = ( + self.workflow_client.projects() # type: ignore + .locations() + .workflows() + .create( + parent=parent, + workflowId=workflow_name, + body={ + "name": full_workflow_name, + "sourceContents": definition, + }, + ) + ) + create_req.execute() + self.logging.info(f"Workflow {workflow_name} has been created!") + + workflow = GCPWorkflow( + workflow_name, + funcs, + benchmark, + code_package.hash, + timeout, + memory, + code_bucket, + ) + else: + # if result is not empty, then function does exists + self.logging.info( + "Workflow {} exists on GCP, update the instance.".format(workflow_name) + ) + + workflow = GCPWorkflow( + name=workflow_name, + functions=funcs, + benchmark=benchmark, + code_package_hash=code_package.hash, + timeout=timeout, + memory=memory, + bucket=code_bucket, + ) + self.update_workflow(workflow, code_package) + + # Add LibraryTrigger to a new function + from sebs.gcp.triggers import WorkflowLibraryTrigger + + trigger = WorkflowLibraryTrigger(workflow_name, self) + trigger.logging_handlers = self.logging_handlers + workflow.add_trigger(trigger) + + return workflow + + def create_workflow_trigger( + self, workflow: Workflow, trigger_type: Trigger.TriggerType + ) -> Trigger: + from sebs.gcp.triggers import WorkflowLibraryTrigger + + if trigger_type == Trigger.TriggerType.HTTP: + raise NotImplementedError("Cannot create http triggers for workflows.") + else: + trigger = WorkflowLibraryTrigger(workflow.name, self) + + trigger.logging_handlers = self.logging_handlers + workflow.add_trigger(trigger) + self.cache_client.update_benchmark(workflow) + return trigger + + def update_workflow(self, workflow: Workflow, code_package: CodePackage): + from sebs.gcp.triggers import HTTPTrigger + + workflow = cast(GCPWorkflow, workflow) + + # Make sure we have a valid workflow benchmark + definition_path = os.path.join(code_package.path, "definition.json") + if not os.path.exists(definition_path): + raise ValueError(f"No workflow definition found for {workflow.name}") + + # First we create a function for each code file + prefix = workflow.name + "___" + code_files = list(code_package.get_code_files(include_config=False)) + func_names = [os.path.splitext(os.path.basename(p))[0] for p in code_files] + funcs = [self.create_function(code_package, prefix + fn) for fn in func_names] + + # Generate workflow definition.json + triggers = [self.create_function_trigger(f, Trigger.TriggerType.HTTP) for f in funcs] + urls = [cast(HTTPTrigger, t).url for t in triggers] + func_triggers = {n: u for (n, u) in zip(func_names, urls)} + gen = GCPGenerator(workflow.name, func_triggers) + gen.parse(definition_path) + definition = gen.generate() + + for map_id, map_def in gen.generate_maps(): + full_workflow_name = GCP.get_full_workflow_name( + self.config.project_name, self.config.region, map_id + ) + patch_req = ( + self.workflow_client.projects() # type: ignore + .locations() + .workflows() + .patch( + name=full_workflow_name, + body={ + "name": full_workflow_name, + "sourceContents": map_def, + }, + ) + ) + patch_req.execute() + self.logging.info("Published new map workflow code.") + + full_workflow_name = GCP.get_full_workflow_name( + self.config.project_name, self.config.region, workflow.name + ) + req = ( + self.workflow_client.projects() # type: ignore + .locations() + .workflows() + .patch( + name=full_workflow_name, + body={"name": full_workflow_name, "sourceContents": definition}, + ) + ) + req.execute() + workflow.functions = funcs + self.logging.info("Published new workflow code and configuration.") + + @staticmethod + def get_full_workflow_name(project_name: str, location: str, workflow_name: str): + return f"projects/{project_name}/locations/{location}/workflows/{workflow_name}" + def prepare_experiment(self, benchmark): logs_bucket = self.storage.add_output_bucket(benchmark, suffix="logs") return logs_bucket @@ -364,7 +566,12 @@ def shutdown(self) -> None: super().shutdown() def download_metrics( - self, function_name: str, start_time: int, end_time: int, requests: dict, metrics: dict + self, + function_name: str, + start_time: int, + end_time: int, + requests: dict, + metrics: dict, ): from google.api_core import exceptions @@ -386,7 +593,8 @@ def wrapper(gen): There shouldn't be problem of waiting for complete results, since logs appear very quickly here. """ - from google.cloud import logging as gcp_logging + + from google.cloud import logging as gcp_logging # type: ignore logging_client = gcp_logging.Client() logger = logging_client.logger("cloudfunctions.googleapis.com%2Fcloud-functions") @@ -502,7 +710,7 @@ def _enforce_cold_start(self, function: Function): return new_version - def enforce_cold_start(self, functions: List[Function], code_package: Benchmark): + def enforce_cold_start(self, functions: List[Function], code_package: CodePackage): new_versions = [] for func in functions: @@ -527,7 +735,9 @@ def enforce_cold_start(self, functions: List[Function], code_package: Benchmark) self.cold_start_counter += 1 - def get_functions(self, code_package: Benchmark, function_names: List[str]) -> List["Function"]: + def get_functions( + self, code_package: CodePackage, function_names: List[str] + ) -> List["Function"]: functions: List["Function"] = [] undeployed_functions_before = [] @@ -572,6 +782,10 @@ def deployment_version(self, func: Function) -> int: status_res = status_req.execute() return int(status_res["versionId"]) + @staticmethod + def get_location(project_name: str, location: str) -> str: + return f"projects/{project_name}/locations/{location}" + # @abstractmethod # def get_invocation_error(self, function_name: str, # start_time: int, end_time: int): diff --git a/sebs/gcp/generator.py b/sebs/gcp/generator.py new file mode 100644 index 00000000..223c0c4b --- /dev/null +++ b/sebs/gcp/generator.py @@ -0,0 +1,101 @@ +import uuid +from typing import Dict, Union, List + +from sebs.faas.fsm import Generator, State, Task, Switch, Map, Repeat, Loop + + +class GCPGenerator(Generator): + def __init__(self, workflow_name: str, func_triggers: Dict[str, str]): + super().__init__() + self._workflow_name = workflow_name + self._func_triggers = func_triggers + self._map_funcs: Dict[str, str] = dict() + + def postprocess(self, payloads: List[dict]) -> dict: + payloads.append({"final": {"return": ["${res}"]}}) + + definition = {"main": {"params": ["res"], "steps": payloads}} + + return definition + + def encode_task(self, state: Task) -> Union[dict, List[dict]]: + url = self._func_triggers[state.func_name] + + return [ + { + state.name: { + "call": "http.post", + "args": {"url": url, "body": "${res}"}, + "result": "res", + } + }, + {"assign_res_" + state.name: {"assign": [{"res": "${res.body}"}]}}, + ] + + def encode_switch(self, state: Switch) -> Union[dict, List[dict]]: + return { + state.name: { + "switch": [self._encode_case(c) for c in state.cases], + "next": state.default, + } + } + + def _encode_case(self, case: Switch.Case) -> dict: + cond = "res." + case.var + " " + case.op + " " + str(case.val) + return {"condition": "${" + cond + "}", "next": case.next} + + def encode_map(self, state: Map) -> Union[dict, List[dict]]: + id = self._workflow_name + "_" + "map" + str(uuid.uuid4())[0:8] + self._map_funcs[id] = self._func_triggers[state.func_name] + + return { + state.name: { + "call": "experimental.executions.map", + "args": {"workflow_id": id, "arguments": "${res." + state.array + "}"}, + "result": "res", + } + } + + def encode_loop(self, state: Loop) -> Union[dict, List[dict]]: + url = self._func_triggers[state.func_name] + + return { + state.name: { + "for": { + "value": "val", + "index": "idx", + "in": "${"+state.array+"}", + "steps": [ + { + "body": { + "call": "http.post", + "args": {"url": url, "body": "${val}"} + } + } + ] + } + } + } + + def generate_maps(self): + for workflow_id, url in self._map_funcs.items(): + yield ( + workflow_id, + self._export_func( + { + "main": { + "params": ["elem"], + "steps": [ + { + "map": { + "call": "http.post", + "args": {"url": url, "body": "${elem}"}, + "result": "elem", + } + }, + {"ret": {"return": "${elem.body}"}}, + ], + } + } + ), + ) diff --git a/sebs/gcp/storage.py b/sebs/gcp/storage.py index 8202cd0e..8c170a90 100644 --- a/sebs/gcp/storage.py +++ b/sebs/gcp/storage.py @@ -2,7 +2,7 @@ import uuid from typing import List -from google.cloud import storage as gcp_storage +from google.cloud import storage as gcp_storage # type: ignore from sebs.cache import Cache from ..faas.storage import PersistentStorage diff --git a/sebs/gcp/triggers.py b/sebs/gcp/triggers.py index 13cc3d6c..9135512c 100644 --- a/sebs/gcp/triggers.py +++ b/sebs/gcp/triggers.py @@ -4,14 +4,17 @@ import time from typing import Dict, Optional # noqa +from google.cloud.workflows.executions_v1beta import ExecutionsClient +from google.cloud.workflows.executions_v1beta.types import Execution + from sebs.gcp.gcp import GCP -from sebs.faas.function import ExecutionResult, Trigger +from sebs.faas.benchmark import ExecutionResult, Trigger class LibraryTrigger(Trigger): - def __init__(self, fname: str, deployment_client: Optional[GCP] = None): + def __init__(self, name: str, deployment_client: Optional[GCP] = None): super().__init__() - self.name = fname + self.name = name self._deployment_client = deployment_client @staticmethod @@ -31,6 +34,18 @@ def deployment_client(self, deployment_client: GCP): def trigger_type() -> Trigger.TriggerType: return Trigger.TriggerType.LIBRARY + def async_invoke(self, payload: dict): + raise NotImplementedError() + + def serialize(self) -> dict: + return {"type": "Library", "name": self.name} + + @classmethod + def deserialize(cls, obj: dict) -> Trigger: + return cls(obj["name"]) + + +class FunctionLibraryTrigger(LibraryTrigger): def sync_invoke(self, payload: dict) -> ExecutionResult: self.logging.info(f"Invoke function {self.name}") @@ -71,15 +86,52 @@ def sync_invoke(self, payload: dict) -> ExecutionResult: gcp_result.parse_benchmark_output(output) return gcp_result - def async_invoke(self, payload: dict): - raise NotImplementedError() - def serialize(self) -> dict: - return {"type": "Library", "name": self.name} +class WorkflowLibraryTrigger(LibraryTrigger): + def sync_invoke(self, payload: dict) -> ExecutionResult: - @staticmethod - def deserialize(obj: dict) -> Trigger: - return LibraryTrigger(obj["name"]) + self.logging.info(f"Invoke workflow {self.name}") + + # Verify that the function is deployed + # deployed = False + # while not deployed: + # if self.deployment_client.is_deployed(self.name): + # deployed = True + # else: + # time.sleep(5) + + # GCP's fixed style for a function name + config = self.deployment_client.config + full_workflow_name = GCP.get_full_workflow_name( + config.project_name, config.region, self.name + ) + + execution_client = ExecutionsClient() + execution = Execution(argument=json.dumps(payload)) + + begin = datetime.datetime.now() + res = execution_client.create_execution(parent=full_workflow_name, execution=execution) + end = datetime.datetime.now() + + gcp_result = ExecutionResult.from_times(begin, end) + + # Wait for execution to finish, then print results. + execution_finished = False + while not execution_finished: + execution = execution_client.get_execution(request={"name": res.name}) + execution_finished = execution.state != Execution.State.ACTIVE + + # If we haven't seen the result yet, wait a second. + if not execution_finished: + time.sleep(10) + elif execution.state == Execution.State.FAILED: + self.logging.error(f"Invocation of {self.name} failed") + self.logging.error(f"Input: {payload}") + gcp_result.stats.failure = True + return gcp_result + + gcp_result.parse_benchmark_execution(execution) + return gcp_result class HTTPTrigger(Trigger): @@ -108,6 +160,6 @@ def async_invoke(self, payload: dict) -> concurrent.futures.Future: def serialize(self) -> dict: return {"type": "HTTP", "url": self.url} - @staticmethod - def deserialize(obj: dict) -> Trigger: + @classmethod + def deserialize(cls, obj: dict) -> Trigger: return HTTPTrigger(obj["url"]) diff --git a/sebs/gcp/workflow.py b/sebs/gcp/workflow.py new file mode 100644 index 00000000..f1846bc4 --- /dev/null +++ b/sebs/gcp/workflow.py @@ -0,0 +1,65 @@ +from typing import List, cast, Optional + +from sebs.faas.benchmark import Workflow +from sebs.gcp.function import GCPFunction +from sebs.gcp.storage import GCPStorage + + +class GCPWorkflow(Workflow): + def __init__( + self, + name: str, + functions: List[GCPFunction], + benchmark: str, + code_package_hash: str, + timeout: int, + memory: int, + bucket: Optional[str] = None, + ): + super().__init__(benchmark, name, code_package_hash) + self.functions = functions + self.timeout = timeout + self.memory = memory + self.bucket = bucket + + @staticmethod + def typename() -> str: + return "GCP.GCPWorkflow" + + def serialize(self) -> dict: + return { + **super().serialize(), + "functions": [f.serialize() for f in self.functions], + "timeout": self.timeout, + "memory": self.memory, + "bucket": self.bucket, + } + + @staticmethod + def deserialize(cached_config: dict) -> "GCPWorkflow": + from sebs.faas.benchmark import Trigger + from sebs.gcp.triggers import WorkflowLibraryTrigger, HTTPTrigger + + funcs = [GCPFunction.deserialize(f) for f in cached_config["functions"]] + ret = GCPWorkflow( + cached_config["name"], + funcs, + cached_config["code_package"], + cached_config["hash"], + cached_config["timeout"], + cached_config["memory"], + cached_config["bucket"], + ) + for trigger in cached_config["triggers"]: + trigger_type = cast( + Trigger, + {"Library": WorkflowLibraryTrigger, "HTTP": HTTPTrigger}.get(trigger["type"]), + ) + assert trigger_type, "Unknown trigger type {}".format(trigger["type"]) + ret.add_trigger(trigger_type.deserialize(trigger)) + return ret + + def code_bucket(self, benchmark: str, storage_client: GCPStorage): + if not self.bucket: + self.bucket, idx = storage_client.add_input_bucket(benchmark) + return self.bucket diff --git a/sebs/local/deployment.py b/sebs/local/deployment.py index d3f0e4b7..66a1d50c 100644 --- a/sebs/local/deployment.py +++ b/sebs/local/deployment.py @@ -26,7 +26,11 @@ def serialize(self, path: str): with open(path, "w") as out: out.write( serialize( - {"functions": self._functions, "storage": self._storage, "inputs": self._inputs} + { + "functions": self._functions, + "storage": self._storage, + "inputs": self._inputs, + } ) ) diff --git a/sebs/local/function.py b/sebs/local/function.py index 8bf408be..397efabb 100644 --- a/sebs/local/function.py +++ b/sebs/local/function.py @@ -2,7 +2,7 @@ import docker import json -from sebs.faas.function import ExecutionResult, Function, Trigger +from sebs.faas.benchmark import ExecutionResult, Function, Trigger class HTTPTrigger(Trigger): @@ -30,14 +30,19 @@ def async_invoke(self, payload: dict) -> concurrent.futures.Future: def serialize(self) -> dict: return {"type": "HTTP", "url": self.url} - @staticmethod - def deserialize(obj: dict) -> Trigger: + @classmethod + def deserialize(cls, obj: dict) -> Trigger: return HTTPTrigger(obj["url"]) class LocalFunction(Function): def __init__( - self, docker_container, port: int, name: str, benchmark: str, code_package_hash: str + self, + docker_container, + port: int, + name: str, + benchmark: str, + code_package_hash: str, ): super().__init__(benchmark, name, code_package_hash) self._instance = docker_container diff --git a/sebs/local/local.py b/sebs/local/local.py index 216f0d41..90f4125b 100644 --- a/sebs/local/local.py +++ b/sebs/local/local.py @@ -11,10 +11,10 @@ from sebs.local.config import LocalConfig from sebs.local.storage import Minio from sebs.local.function import LocalFunction -from sebs.faas.function import Function, ExecutionResult, Trigger +from sebs.faas.benchmark import Benchmark, Function, Workflow, ExecutionResult, Trigger from sebs.faas.storage import PersistentStorage from sebs.faas.system import System -from sebs.benchmark import Benchmark +from sebs.code_package import CodePackage class Local(System): @@ -33,6 +33,10 @@ def typename(): def function_type() -> "Type[Function]": return LocalFunction + @staticmethod + def workflow_type() -> "Type[Workflow]": + raise NotImplementedError() + @property def config(self) -> LocalConfig: return self._config @@ -115,13 +119,15 @@ def shutdown(self): benchmark: benchmark name """ - def package_code(self, directory: str, language_name: str, benchmark: str) -> Tuple[str, int]: + def package_code( + self, code_package: CodePackage, directory: str, is_workflow: bool + ) -> Tuple[str, int]: CONFIG_FILES = { "python": ["handler.py", "requirements.txt", ".python_packages"], "nodejs": ["handler.js", "package.json", "node_modules"], } - package_config = CONFIG_FILES[language_name] + package_config = CONFIG_FILES[code_package.language_name] function_dir = os.path.join(directory, "function") os.makedirs(function_dir) # move all files to 'function' except handler.py @@ -136,10 +142,11 @@ def package_code(self, directory: str, language_name: str, benchmark: str) -> Tu return directory, bytes_size - def create_function(self, code_package: Benchmark, func_name: str) -> "LocalFunction": + def create_function(self, code_package: CodePackage, func_name: str) -> "LocalFunction": home_dir = os.path.join( - "/home", self._system_config.username(self.name(), code_package.language_name) + "/home", + self._system_config.username(self.name(), code_package.language_name), ) container_name = "{}:run.local.{}.{}".format( self._system_config.docker_repository(), @@ -157,7 +164,10 @@ def create_function(self, code_package: Benchmark, func_name: str) -> "LocalFunc image=container_name, command=f"python3 server.py {self.DEFAULT_PORT}", volumes={ - code_package.code_location: {"bind": os.path.join(home_dir, "code"), "mode": "ro"} + code_package.code_location: { + "bind": os.path.join(home_dir, "code"), + "mode": "ro", + } }, environment=environment, # FIXME: make CPUs configurable @@ -176,7 +186,11 @@ def create_function(self, code_package: Benchmark, func_name: str) -> "LocalFunc # tty=True, ) func = LocalFunction( - container, self.DEFAULT_PORT, func_name, code_package.benchmark, code_package.hash + container, + self.DEFAULT_PORT, + func_name, + code_package.name, + code_package.hash, ) self.logging.info( f"Started {func_name} function at container {container.id} , running on {func._url}" @@ -187,7 +201,7 @@ def create_function(self, code_package: Benchmark, func_name: str) -> "LocalFunc FIXME: restart Docker? """ - def update_function(self, function: Function, code_package: Benchmark): + def update_function(self, function: Function, code_package: CodePackage): pass """ @@ -195,7 +209,7 @@ def update_function(self, function: Function, code_package: Benchmark): There's only one trigger - HTTP. """ - def create_trigger(self, func: Function, trigger_type: Trigger.TriggerType) -> Trigger: + def create_function_trigger(self, func: Function, trigger_type: Trigger.TriggerType) -> Trigger: from sebs.local.function import HTTPTrigger function = cast(LocalFunction, func) @@ -206,10 +220,10 @@ def create_trigger(self, func: Function, trigger_type: Trigger.TriggerType) -> T raise RuntimeError("Not supported!") function.add_trigger(trigger) - self.cache_client.update_function(function) + self.cache_client.update_benchmark(function) return trigger - def cached_function(self, function: Function): + def cached_benchmark(self, benchmark: Benchmark): pass def download_metrics( @@ -222,19 +236,30 @@ def download_metrics( ): pass - def enforce_cold_start(self, functions: List[Function], code_package: Benchmark): + def enforce_cold_start(self, functions: List[Function], code_package: CodePackage): raise NotImplementedError() @staticmethod - def default_function_name(code_package: Benchmark) -> str: + def default_benchmark_name(code_package: CodePackage) -> str: # Create function name func_name = "{}-{}-{}".format( - code_package.benchmark, + code_package.name, code_package.language_name, - code_package.benchmark_config.memory, + code_package.config.memory, ) return func_name @staticmethod def format_function_name(func_name: str) -> str: return func_name + + def create_workflow(self, code_package: CodePackage, workflow_name: str) -> Workflow: + raise NotImplementedError() + + def create_workflow_trigger( + self, workflow: Workflow, trigger_type: Trigger.TriggerType + ) -> Trigger: + raise NotImplementedError() + + def update_workflow(self, workflow: Workflow, code_package: CodePackage): + raise NotImplementedError() diff --git a/sebs/local/storage.py b/sebs/local/storage.py index c34f4c0d..9af64149 100644 --- a/sebs/local/storage.py +++ b/sebs/local/storage.py @@ -83,7 +83,10 @@ def stop(self): def get_connection(self): return minio.Minio( - self._url, access_key=self._access_key, secret_key=self._secret_key, secure=False + self._url, + access_key=self._access_key, + secret_key=self._secret_key, + secure=False, ) def _create_bucket(self, name: str, buckets: List[str] = []): diff --git a/sebs/regression.py b/sebs/regression.py index 37be04aa..9c7e5bf8 100644 --- a/sebs/regression.py +++ b/sebs/regression.py @@ -4,7 +4,7 @@ from time import sleep from typing import cast, Dict, Optional, Set, TYPE_CHECKING -from sebs.faas.function import Trigger +from sebs.faas.benchmark import Trigger if TYPE_CHECKING: from sebs import SeBS @@ -48,7 +48,7 @@ def test(self): replace_existing=experiment_config.update_storage ) func = deployment_client.get_function( - benchmark, deployment_client.default_function_name(benchmark) + benchmark, deployment_client.default_benchmark_name(benchmark) ) input_config = benchmark.prepare_input(storage=storage, size="test") diff --git a/sebs/sebs.py b/sebs/sebs.py index 4562c7bb..e19ae9cd 100644 --- a/sebs/sebs.py +++ b/sebs/sebs.py @@ -5,7 +5,7 @@ from sebs.local import Local from sebs.cache import Cache from sebs.config import SeBSConfig -from sebs.benchmark import Benchmark +from sebs.code_package import CodePackage from sebs.faas.system import System as FaaSSystem from sebs.faas.config import Config from sebs.utils import has_platform, LoggingHandlers, LoggingBase @@ -149,8 +149,8 @@ def get_benchmark( deployment: FaaSSystem, config: ExperimentConfig, logging_filename: Optional[str] = None, - ) -> Benchmark: - benchmark = Benchmark( + ) -> CodePackage: + code_package = CodePackage( name, deployment.name(), config, @@ -159,10 +159,10 @@ def get_benchmark( self.cache_client, self.docker_client, ) - benchmark.logging_handlers = self.generate_logging_handlers( + code_package.logging_handlers = self.generate_logging_handlers( logging_filename=logging_filename ) - return benchmark + return code_package def shutdown(self): self.cache_client.shutdown() diff --git a/sebs/utils.py b/sebs/utils.py index eff58511..cef1f6cf 100644 --- a/sebs/utils.py +++ b/sebs/utils.py @@ -7,6 +7,8 @@ import uuid from typing import List, Optional, TextIO, Union +from redis import Redis + PROJECT_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir) PACK_CODE_APP = "pack_code_{}.sh" @@ -85,6 +87,46 @@ def configure_logging(): logging.getLogger(name).setLevel(logging.ERROR) +def replace_string_in_file(path: str, from_str: str, to_str: str): + with open(path, "rt") as f: + data = f.read() + + data = data.replace(from_str, to_str) + + with open(path, "wt") as f: + f.write(data) + + +def connect_to_redis_cache(host: str): + redis = Redis(host=host, port=6379, decode_responses=True, socket_connect_timeout=10) + redis.ping() + + return redis + + +def download_measurements(redis: Redis, workflow_name: str, after: float, **static_args): + payloads = [] + + for key in redis.scan_iter(match=f"{workflow_name}/*"): + assert key[: len(workflow_name)] == workflow_name + + payload = redis.get(key) + redis.delete(key) + + if payload: + try: + payload = json.loads(payload) + + # make sure only measurements from our benchmark are saved + if payload["start"] > after: + payload = {**payload, **static_args} + payloads.append(payload) + except json.decoder.JSONDecodeError: + print(f"Failed to decode payload: {payload}") + + return payloads + + # def configure_logging(verbose: bool = False, output_dir: Optional[str] = None): # logging_format = "%(asctime)s,%(msecs)d %(levelname)s %(name)s: %(message)s" # logging_date_format = "%H:%M:%S" @@ -129,7 +171,7 @@ def configure_logging(): """ -def find_benchmark(benchmark: str, path: str): +def find_package_code(benchmark: str, path: str): benchmarks_dir = os.path.join(PROJECT_DIR, path) benchmark_path = find(benchmark, benchmarks_dir) return benchmark_path diff --git a/tests/aws/create_function.py b/tests/aws/create_function.py index e672cc89..3fc20dbb 100644 --- a/tests/aws/create_function.py +++ b/tests/aws/create_function.py @@ -39,7 +39,7 @@ class AWSCreateFunction(unittest.TestCase): "nodejs": ["handler.js", "function/storage.js", "package.json", "node_modules/"] } benchmark = "110.dynamic-html" - function_name_suffixes = [] + function_name_suffixes = [] def setUp(self): self.tmp_dir = tempfile.TemporaryDirectory() @@ -68,7 +68,7 @@ def setUpClass(cls): benchmark = cls.client.get_benchmark( cls.benchmark, cls.tmp_dir.name, deployment_client, experiment_config ) - func_name = deployment_client.default_function_name(benchmark) + func_name = deployment_client.default_benchmark_name(benchmark) for suffix in cls.function_name_suffixes: deployment_client.delete_function(func_name + suffix) @@ -91,12 +91,12 @@ def tearDownClass(cls): benchmark = cls.client.get_benchmark( cls.benchmark, cls.tmp_dir.name, deployment_client, experiment_config ) - func_name = deployment_client.default_function_name(benchmark) + func_name = deployment_client.default_benchmark_name(benchmark) for suffix in cls.function_name_suffixes: deployment_client.delete_function(func_name + suffix) def check_function( - self, language: str, package: sebs.benchmark.Benchmark, files: List[str] + self, language: str, package: sebs.code_package.CodePackage, files: List[str] ): filename, file_extension = os.path.splitext(package.code_location) self.assertEqual(file_extension, ".zip") @@ -131,7 +131,7 @@ def test_create_function(self): for language in ["python", "nodejs"]: benchmark, deployment_client, experiment_config = self.generate_benchmark(tmp_dir, language) - func_name = deployment_client.default_function_name(benchmark) + self.function_name_suffixes[0] + func_name = deployment_client.default_benchmark_name(benchmark) + self.function_name_suffixes[0] func = deployment_client.get_function(benchmark, func_name) self.assertIsInstance(func, sebs.aws.LambdaFunction) self.assertEqual(func.name, func_name) @@ -143,7 +143,7 @@ def test_retrieve_cache(self): benchmark, deployment_client, experiment_config = self.generate_benchmark(tmp_dir, language) # generate default variant - func_name = deployment_client.default_function_name(benchmark) + self.function_name_suffixes[1] + func_name = deployment_client.default_benchmark_name(benchmark) + self.function_name_suffixes[1] func = deployment_client.get_function(benchmark, func_name) timestamp = os.path.getmtime(benchmark.code_location) self.assertIsInstance(func, sebs.aws.LambdaFunction) @@ -175,7 +175,7 @@ def test_rebuild_function(self): benchmark, deployment_client, experiment_config = self.generate_benchmark(tmp_dir, language) # generate default variant - func_name = deployment_client.default_function_name(benchmark) + self.function_name_suffixes[2] + func_name = deployment_client.default_benchmark_name(benchmark) + self.function_name_suffixes[2] func = deployment_client.get_function(benchmark, func_name) timestamp = os.path.getmtime(benchmark.code_location) self.assertIsInstance(func, sebs.aws.LambdaFunction) @@ -209,7 +209,7 @@ def test_update_function(self): benchmark, deployment_client, experiment_config = self.generate_benchmark(tmp_dir, language) # generate default variant - func_name = deployment_client.default_function_name(benchmark) + self.function_name_suffixes[3] + func_name = deployment_client.default_benchmark_name(benchmark) + self.function_name_suffixes[3] func = deployment_client.get_function(benchmark, func_name) timestamp = os.path.getmtime(benchmark.code_location) self.assertIsInstance(func, sebs.aws.LambdaFunction) diff --git a/tests/aws/invoke_function_http.py b/tests/aws/invoke_function_http.py index c603fe88..39f0776a 100644 --- a/tests/aws/invoke_function_http.py +++ b/tests/aws/invoke_function_http.py @@ -42,8 +42,8 @@ def test_invoke_sync_python(self): bench_input = benchmark.prepare_input( storage=deployment_client.get_storage(), size="test" ) - func = deployment_client.get_function(benchmark, '{}-http'.format(sebs.aws.AWS.default_function_name(benchmark))) - from sebs.faas.function import Trigger + func = deployment_client.get_function(benchmark, '{}-http'.format(sebs.aws.AWS.default_benchmark_name(benchmark))) + from sebs.faas.benchmark import Trigger deployment_client.create_trigger(func, Trigger.TriggerType.HTTP) self.invoke_sync(func, bench_input) @@ -70,8 +70,8 @@ def test_invoke_sync_nodejs(self): bench_input = benchmark.prepare_input( storage=deployment_client.get_storage(), size="test" ) - func = deployment_client.get_function(benchmark, '{}-http'.format(sebs.aws.AWS.default_function_name(benchmark))) - from sebs.faas.function import Trigger + func = deployment_client.get_function(benchmark, '{}-http'.format(sebs.aws.AWS.default_benchmark_name(benchmark))) + from sebs.faas.benchmark import Trigger deployment_client.create_trigger(func, Trigger.TriggerType.HTTP) self.invoke_sync(func, bench_input) diff --git a/tests/regression.py b/tests/regression.py index 4ec40426..14446502 100755 --- a/tests/regression.py +++ b/tests/regression.py @@ -63,7 +63,7 @@ def test(self): input_config = benchmark.prepare_input(storage=storage, size="test") func = deployment_client.get_function( benchmark, - deployment_client.default_function_name(benchmark) + deployment_client.default_benchmark_name(benchmark) ) ret = func.triggers[0].sync_invoke(input_config) if ret.stats.failure: diff --git a/tools/create_azure_credentials.py b/tools/create_azure_credentials.py index 50260745..8c93a60a 100755 --- a/tools/create_azure_credentials.py +++ b/tools/create_azure_credentials.py @@ -53,7 +53,7 @@ print(out.decode()) else: credentials = json.loads(out.decode()) - print('Created service principal {}'.format(credentials['name'])) + print('Created service principal {}'.format(credentials['displayName'])) print('AZURE_SECRET_APPLICATION_ID = {}'.format(credentials['appId'])) print('AZURE_SECRET_TENANT = {}'.format(credentials['tenant'])) print('AZURE_SECRET_PASSWORD = {}'.format(credentials['password']))