updated support for benchmark language versions

benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json.18

@@ -0,0 +1,10 @@
+{
+  "name": "",
+  "version": "1.0.0",
+  "description": "",
+  "author": "",
+  "license": "",
+  "dependencies": {
+    "sharp": "^0.33"
+  }
+}

benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json.20

@@ -0,0 +1,10 @@
+{
+  "name": "",
+  "version": "1.0.0",
+  "description": "",
+  "author": "",
+  "license": "",
+  "dependencies": {
+    "sharp": "^0.33"
+  }
+}

benchmarks/200.multimedia/210.thumbnailer/python/requirements.txt.3.10

@@ -0,0 +1 @@
+pillow==10.3.0

benchmarks/200.multimedia/210.thumbnailer/python/requirements.txt.3.11

@@ -0,0 +1 @@
+pillow==10.3.0

benchmarks/200.multimedia/210.thumbnailer/python/requirements.txt.3.12

@@ -0,0 +1 @@
+pillow==10.3.0

benchmarks/400.inference/411.image-recognition/python/requirements.txt.3.10

@@ -0,0 +1,5 @@
+pillow==10.3.0
+https://download.pytorch.org/whl/cpu/torch-1.11.0%2Bcpu-cp310-cp310-linux_x86_64.whl
+torchvision==0.12
+# prevent installing numpy 2.0
+numpy==1.22.0

benchmarks/400.inference/411.image-recognition/python/requirements.txt.3.11

@@ -0,0 +1,5 @@
+pillow==10.3.0
+https://download.pytorch.org/whl/cpu/torch-2.0.0%2Bcpu-cp311-cp311-linux_x86_64.whl
+torchvision==0.15.1
+# prevent installing numpy 2.0
+numpy==1.24.0

benchmarks/500.scientific/501.graph-pagerank/python/requirements.txt.3.10

@@ -0,0 +1 @@
+igraph==0.11.4

benchmarks/500.scientific/501.graph-pagerank/python/requirements.txt.3.11

@@ -0,0 +1 @@
+igraph==0.11.4

benchmarks/500.scientific/501.graph-pagerank/python/requirements.txt.3.12

@@ -0,0 +1 @@
+igraph==0.11.4

benchmarks/500.scientific/502.graph-mst/python/requirements.txt.3.10

@@ -0,0 +1 @@
+igraph==0.11.4

benchmarks/500.scientific/502.graph-mst/python/requirements.txt.3.11

@@ -0,0 +1 @@
+igraph==0.11.4

benchmarks/500.scientific/502.graph-mst/python/requirements.txt.3.12

@@ -0,0 +1 @@
+igraph==0.11.4

benchmarks/500.scientific/503.graph-bfs/python/requirements.txt.3.10

@@ -0,0 +1 @@
+igraph==0.11.4

benchmarks/500.scientific/503.graph-bfs/python/requirements.txt.3.11

@@ -0,0 +1 @@
+igraph==0.11.4

benchmarks/500.scientific/503.graph-bfs/python/requirements.txt.3.12

@@ -0,0 +1 @@
+igraph==0.11.4

config/systems.json

@@ -17,8 +17,8 @@
     "languages": {
       "python": {
         "base_images": {
-          "3.7": "python:3.7-slim",
-          "3.8": "python:3.8-slim"
+          "3.8": "python:3.8-slim",
+          "3.9": "python:3.9-slim"
         },
         "images": [
           "run",
@@ -34,8 +34,8 @@
       },
       "nodejs": {
         "base_images": {
-          "12": "node:12-slim",
-          "14": "node:14-slim"
+          "18": "node:18-slim",
+          "20": "node:20-slim"
         },
         "images": [
           "run",
@@ -55,15 +55,11 @@
     "languages": {
       "python": {
         "base_images": {
+          "3.11": "amazon/aws-lambda-python:3.11",
+          "3.10": "amazon/aws-lambda-python:3.10",
           "3.9": "amazon/aws-lambda-python:3.9",
-          "3.8": "amazon/aws-lambda-python:3.8",
-          "3.7": "amazon/aws-lambda-python:3.7"
+          "3.8": "amazon/aws-lambda-python:3.8"
         },
-        "versions": [
-          "3.7",
-          "3.8",
-          "3.9"
-        ],
         "images": [
           "build"
         ],
@@ -77,12 +73,8 @@
       },
       "nodejs": {
         "base_images": {
-          "14": "amazon/aws-lambda-nodejs:14",
           "16": "amazon/aws-lambda-nodejs:16"
         },
-        "versions": [
-          "14", "16"
-        ],
         "images": [
           "build"
         ],
@@ -104,7 +96,9 @@
         "base_images": {
           "3.7": "mcr.microsoft.com/azure-functions/python:3.0-python3.7",
           "3.8": "mcr.microsoft.com/azure-functions/python:3.0-python3.8",
-          "3.9": "mcr.microsoft.com/azure-functions/python:3.0-python3.9"
+          "3.9": "mcr.microsoft.com/azure-functions/python:3.0-python3.9",
+          "3.10": "mcr.microsoft.com/azure-functions/python:4-python3.10",
+          "3.11": "mcr.microsoft.com/azure-functions/python:4-python3.11"
         },
         "images": [
           "build"
@@ -122,7 +116,9 @@
       },
       "nodejs": {
         "base_images": {
-          "14": "mcr.microsoft.com/azure-functions/node:3.0-node14"
+          "16": "mcr.microsoft.com/azure-functions/node:4-node16",
+          "18": "mcr.microsoft.com/azure-functions/node:4-node18",
+          "20": "mcr.microsoft.com/azure-functions/node:4-node20"
         },
         "images": [
           "build"
@@ -152,7 +148,10 @@
         "base_images": {
           "3.7": "ubuntu:22.04",
           "3.8": "ubuntu:22.04",
-          "3.9": "ubuntu:22.04"
+          "3.9": "ubuntu:22.04",
+          "3.10": "ubuntu:22.04",
+          "3.11": "ubuntu:22.04",
+          "3.12": "ubuntu:22.04"
         },
         "images": [
           "build"
@@ -170,10 +169,12 @@
       },
       "nodejs": {
         "base_images": {
-          "10": "gcr.io/google-appengine/nodejs",
-          "12": "gcr.io/google-appengine/nodejs",
-          "14": "gcr.io/google-appengine/nodejs",
-          "16": "gcr.io/google-appengine/nodejs"
+          "10": "ubuntu:18.04",
+          "12": "ubuntu:18.04",
+          "14": "ubuntu:18.04",
+          "16": "ubuntu:18.04",
+          "18": "ubuntu:22.04",
+          "20": "ubuntu:22.04"
         },
         "images": [
           "build"
@@ -197,7 +198,9 @@
       "python": {
         "base_images": {
           "3.7": "openwhisk/action-python-v3.7",
-          "3.9": "openwhisk/action-python-v3.9"
+          "3.9": "openwhisk/action-python-v3.9",
+          "3.10": "openwhisk/action-python-v3.10",
+          "3.11": "openwhisk/action-python-v3.11"
         },
         "images": [
           "function"
@@ -217,7 +220,10 @@
       "nodejs": {
         "base_images": {
           "10": "openwhisk/action-nodejs-v10",
-          "12": "openwhisk/action-nodejs-v12"
+          "12": "openwhisk/action-nodejs-v12",
+          "14": "openwhisk/action-nodejs-v14",
+          "18": "openwhisk/action-nodejs-v18",
+          "20": "openwhisk/action-nodejs-v20"
         },
         "images": [
           "function"

dockerfiles/gcp/nodejs/Dockerfile.build

@@ -1,10 +1,12 @@
 ARG BASE_IMAGE
 FROM ${BASE_IMAGE}
 ARG VERSION
-ENV HOME=/home/${USER}
+ENV NVM_DIR=/nvm
 
-RUN install_node --ignore-verification-failure v${VERSION}
-RUN apt-get update && apt-get install -y gosu
+#RUN install_node --ignore-verification-failure v${VERSION}
+RUN apt-get update && apt-get install -y gosu wget
+RUN mkdir -p ${NVM_DIR} && wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash
+RUN . ${NVM_DIR}/nvm.sh && nvm install ${VERSION} && nvm alias default ${VERSION} && nvm use default
 
 RUN mkdir -p /sebs/
 COPY dockerfiles/nodejs_installer.sh /sebs/installer.sh
@@ -16,3 +18,4 @@ ENV PATH=/usr/sbin:$PATH
 ENV SCRIPT_FILE=/mnt/function/package.sh
 CMD /bin/bash /sebs/installer.sh
 ENTRYPOINT ["/sebs/entrypoint.sh"]
+

dockerfiles/nodejs_installer.sh

@@ -1,3 +1,6 @@
 #!/bin/bash
 
+if [ -f $FILE ]; then
+  . /nvm/nvm.sh
+fi
 cd /mnt/function && npm install && rm -rf package-lock.json

docs/benchmarks.md

@@ -26,7 +26,7 @@ Below, we discuss the most important implementation details of each benchmark. F
 > Benchmark `411.image-recognition` does not work on AWS with Python 3.9 due to excessive code size. While it is possible to ship the benchmark by zipping `torchvision` and `numpy` (see `benchmarks/400.inference/411.image-recognition/python/package.sh`), this significantly affects cold startup. On the lowest supported memory configuration of 512 MB, the cold startup can reach 30 seconds, making HTTP trigger unusable due to 30 second timeout of API gateway. In future, we might support Docker-based deployment on AWS that are not affected by code size limitations.
 
 > [!WARNING] 
-> Benchmark `411.image-recognition` does not work on GCP with Python 3.8 and 3.9 due to excessive code size. To the best of our knowledge, there is no way of circumventing that limit, as Google Cloud offers neither layers nor custom Docker images.
+> Benchmark `411.image-recognition` does not work on GCP with Python 3.8+ due to excessive code size. To the best of our knowledge, there is no way of circumventing that limit, as Google Cloud offers neither layers nor custom Docker images.
 
 ## Webapps
 

sebs/azure/azure.py

@@ -4,6 +4,7 @@
 import os
 import shutil
 import time
+import uuid
 from typing import cast, Dict, List, Optional, Set, Tuple, Type  # noqa
 
 import docker
@@ -202,7 +203,7 @@ def package_code(
             "version": "2.0",
             "extensionBundle": {
                 "id": "Microsoft.Azure.Functions.ExtensionBundle",
-                "version": "[1.*, 2.0.0)",
+                "version": "[4.0.0, 5.0.0)",
             },
         }
         json.dump(default_host_json, open(os.path.join(directory, "host.json"), "w"), indent=2)
@@ -215,6 +216,7 @@ def publish_function(
         self,
         function: Function,
         code_package: Benchmark,
+        container_dest: str,
         repeat_on_failure: bool = False,
     ) -> str:
         success = False
@@ -223,7 +225,7 @@ def publish_function(
         while not success:
             try:
                 ret = self.cli_instance.execute(
-                    "bash -c 'cd /mnt/function "
+                    f"bash -c 'cd {container_dest} "
                     "&& func azure functionapp publish {} --{} --no-build'".format(
                         function.name, self.AZURE_RUNTIMES[code_package.language_name]
                     )
@@ -286,8 +288,8 @@ def publish_function(
     def update_function(self, function: Function, code_package: Benchmark):
 
         # Mount code package in Docker instance
-        self._mount_function_code(code_package)
-        url = self.publish_function(function, code_package, True)
+        container_dest = self._mount_function_code(code_package)
+        url = self.publish_function(function, code_package, container_dest, True)
 
         trigger = HTTPTrigger(url, self.config.resources.data_storage_account(self.cli_instance))
         trigger.logging_handlers = self.logging_handlers
@@ -299,8 +301,10 @@ def update_function_configuration(self, function: Function, code_package: Benchm
             "Updating function's memory and timeout configuration is not supported."
         )
 
-    def _mount_function_code(self, code_package: Benchmark):
-        self.cli_instance.upload_package(code_package.code_location, "/mnt/function/")
+    def _mount_function_code(self, code_package: Benchmark) -> str:
+        dest = os.path.join("/mnt", "function", uuid.uuid4().hex)
+        self.cli_instance.upload_package(code_package.code_location, dest)
+        return dest
 
     def default_function_name(self, code_package: Benchmark) -> str:
         """
@@ -366,6 +370,7 @@ def create_function(self, code_package: Benchmark, func_name: str) -> AzureFunct
                             " --os-type Linux --consumption-plan-location {region} "
                             " --runtime {runtime} --runtime-version {runtime_version} "
                             " --name {func_name} --storage-account {storage_account}"
+                            " --functions-version 4 "
                         ).format(**config)
                     )
                     self.logging.info("Azure: Created function app {}".format(func_name))

sebs/azure/cli.py

@@ -85,10 +85,24 @@ def login(self, appId: str, tenant: str, password: str) -> bytes:
         return result
 
     def upload_package(self, directory: str, dest: str):
+
+        """
+        This is not an efficient and memory-intensive implementation.
+        So far, we didn't have very large functions that require many gigabytes.
+
+        Since docker-py does not support a straightforward copy, and we can't
+        put_archive in chunks.
+
+        If we end up having problems because of the archive size, there are two
+        potential solutions:
+        (1) manually call docker cp and decompress
+        (2) commit the docker container and restart with a new mount volume.
+        """
         handle = io.BytesIO()
         with tarfile.open(fileobj=handle, mode="w:gz") as tar:
             for f in os.listdir(directory):
                 tar.add(os.path.join(directory, f), arcname=f)
+        # shutil.make_archive(, 'zip', directory)
         # move to the beginning of memory before writing
         handle.seek(0)
         self.execute("mkdir -p {}".format(dest))

sebs/regression.py

@@ -126,6 +126,7 @@ def get_deployment(self, benchmark_name):
                 self.client.output_dir, f"regression_{deployment_name}_{benchmark_name}.log"
             ),
         )
+
         with AWSTestSequencePython.lock:
             deployment_client.initialize(resource_prefix="regression")
         return deployment_client
@@ -229,7 +230,8 @@ def get_deployment(self, benchmark_name):
             cloud_config,
             logging_filename=f"regression_{deployment_name}_{benchmark_name}.log",
         )
-        deployment_client.initialize()
+        with GCPTestSequencePython.lock:
+            deployment_client.initialize(resource_prefix="regression")
         return deployment_client
 
 
@@ -247,7 +249,8 @@ def get_deployment(self, benchmark_name):
             cloud_config,
             logging_filename=f"regression_{deployment_name}_{benchmark_name}.log",
         )
-        deployment_client.initialize()
+        with GCPTestSequenceNodejs.lock:
+            deployment_client.initialize(resource_prefix="regression")
         return deployment_client
 
 
@@ -322,7 +325,12 @@ def filter_out_benchmarks(
     benchmark: str, deployment_name: str, language: str, language_version: str
 ) -> bool:
 
-    if deployment_name == "aws" and language == "python" and language_version == "3.9":
+    if (deployment_name == "aws" and language == "python"
+            and language_version in ["3.9", "3.10", "3.11"]):
+        return "411.image-recognition" not in benchmark
+
+    if (deployment_name == "gcp" and language == "python"
+            and language_version in ["3.8", "3.9", "3.10", "3.11", "3.12"]):
         return "411.image-recognition" not in benchmark
 
     return True