From b9655b0a5adf942cf8b79bd024d5e07a7a39df79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20Juli=C3=A1n?= Date: Thu, 25 Apr 2024 12:14:04 +0200 Subject: [PATCH 01/12] [EBPF-474] Improve kernel information in platforms.json (#24097) * Allow downloading images for all architectures * Add update-platform-info task * Improve output of kmt.ls * Adapt to new platforms.json structure * Use guestfs python package * Ignore rescue images when detecting kernel * Ignore empty version names * Fix kernel detection * Use dynamic distribution mappings * Update platform info * Remove empty alternative names * Better help * Fix archs for custom kernels * Fix import order * Fix format * Use manifest files * Improve sorting in kmt.ls * Update platforms.json * Warn when some images have not been updated * Update platforms.json * Update to latest images * Raise an error when image is missing * Fix platforms.json file * Add kmt.validate-platform-info task * review comments * Improve kmt.ls * Show an error when kernels are missing * Fix image names in CI * Switch Amazon 2023 version * Use master version for AL2023 * test with updated names * Update Amazon 2023 name * Add --exclude-matching argument to update-platform-info task * Update platforms.json --- .../kernel_matrix_testing/security_agent.yml | 8 +- .../kernel_matrix_testing/system_probe.yml | 16 +- tasks/kernel_matrix_testing/download.py | 28 +- tasks/kernel_matrix_testing/types.py | 14 +- tasks/kernel_matrix_testing/vars.py | 1 + tasks/kernel_matrix_testing/vmconfig.py | 180 +++++--- tasks/kmt.py | 159 ++++++- .../system-probe/config/platforms.json | 411 ++++++++++++++++-- 8 files changed, 688 insertions(+), 129 deletions(-) diff --git a/.gitlab/kernel_matrix_testing/security_agent.yml b/.gitlab/kernel_matrix_testing/security_agent.yml index 372f00e6412906..8f8cb0d0b2b349 100644 --- a/.gitlab/kernel_matrix_testing/security_agent.yml +++ b/.gitlab/kernel_matrix_testing/security_agent.yml @@ -147,8 +147,8 @@ kmt_run_secagent_tests_x64: - "ubuntu_20.04" - "ubuntu_22.04" - "ubuntu_23.10" - - "amzn_5.4" - - "amzn_5.10" + - "amazon_5.4" + - "amazon_5.10" - "fedora_37" - "fedora_38" - "debian_10" @@ -181,8 +181,8 @@ kmt_run_secagent_tests_arm64: - TAG: - "ubuntu_22.04" - "ubuntu_23.10" - - "amzn_5.4" - - "amzn_5.10" + - "amazon_5.4" + - "amazon_5.10" - "fedora_37" - "fedora_38" - "debian_11" diff --git a/.gitlab/kernel_matrix_testing/system_probe.yml b/.gitlab/kernel_matrix_testing/system_probe.yml index 68229f386177cc..2f8f570842649e 100644 --- a/.gitlab/kernel_matrix_testing/system_probe.yml +++ b/.gitlab/kernel_matrix_testing/system_probe.yml @@ -225,15 +225,15 @@ kmt_run_sysprobe_tests_x64: - "ubuntu_20.04" - "ubuntu_22.04" - "ubuntu_23.10" - - "amzn_4.14" - - "amzn_5.4" - - "amzn_5.10" + - "amazon_4.14" + - "amazon_5.4" + - "amazon_5.10" - "fedora_37" - "fedora_38" - "debian_10" - "debian_11" - "debian_12" - - "centos_79" + - "centos_7.9" - "centos_8" TEST_SET: ["no_tracersuite", "only_tracersuite"] after_script: @@ -260,15 +260,15 @@ kmt_run_sysprobe_tests_arm64: - "ubuntu_20.04" - "ubuntu_22.04" - "ubuntu_23.10" - - "amzn_4.14" - - "amzn_5.4" - - "amzn_5.10" + - "amazon_4.14" + - "amazon_5.4" + - "amazon_5.10" - "fedora_37" - "fedora_38" - "debian_10" - "debian_11" - "debian_12" - - "centos_79" + - "centos_7.9" - "centos_8" TEST_SET: ["no_tracersuite", "only_tracersuite"] after_script: diff --git a/tasks/kernel_matrix_testing/download.py b/tasks/kernel_matrix_testing/download.py index 0ee38d20025839..9ac1f9d5bfe155 100644 --- a/tasks/kernel_matrix_testing/download.py +++ b/tasks/kernel_matrix_testing/download.py @@ -3,7 +3,7 @@ import os import platform import tempfile -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING from invoke.context import Context @@ -18,7 +18,7 @@ requests = None if TYPE_CHECKING: - from tasks.kernel_matrix_testing.types import PathOrStr + from tasks.kernel_matrix_testing.types import Arch, PathOrStr def requires_update(url_base: str, rootfs_dir: PathOrStr, image: str, branch: str): @@ -41,23 +41,27 @@ def requires_update(url_base: str, rootfs_dir: PathOrStr, image: str, branch: st return False -def download_rootfs(ctx: Context, rootfs_dir: PathOrStr, vmconfig_template_name: str): +def download_rootfs(ctx: Context, rootfs_dir: PathOrStr, vmconfig_template_name: str, arch: Arch | None = None): platforms = get_platforms() vmconfig_template = get_vmconfig_template(vmconfig_template_name) url_base = platforms["url_base"] - arch = arch_mapping[platform.machine()] - to_download: List[str] = list() - file_ls: List[str] = list() + if arch is None: + arch = arch_mapping[platform.machine()] + to_download: list[str] = list() + file_ls: list[str] = list() branch_mapping: dict[str, str] = dict() for tag in platforms[arch]: - path = os.path.basename(platforms[arch][tag]) + platinfo = platforms[arch][tag] + if "image" not in platinfo: + raise Exit("image is not defined in platform info") + path = os.path.basename(platinfo["image"]) if path.endswith(".xz"): path = path[: -len(".xz")] - branch_mapping[path] = os.path.dirname(platforms[arch][tag]) or "master" + branch_mapping[path] = platinfo.get('image_version', 'master') file_ls.append(os.path.basename(path)) # if file does not exist download it. @@ -134,7 +138,11 @@ def download_rootfs(ctx: Context, rootfs_dir: PathOrStr, vmconfig_template_name: raise Exit("Failed to set permissions 0766 to rootfs") -def update_rootfs(ctx: Context, rootfs_dir: PathOrStr, vmconfig_template: str): - download_rootfs(ctx, rootfs_dir, vmconfig_template) +def update_rootfs(ctx: Context, rootfs_dir: PathOrStr, vmconfig_template: str, all_archs: bool = False): + if all_archs: + arch_ls: list[Arch] = ["x86_64", "arm64"] + for arch in arch_ls: + info(f"[+] Updating root filesystem for {arch}") + download_rootfs(ctx, rootfs_dir, vmconfig_template, arch) info("[+] Root filesystem and bootables images updated") diff --git a/tasks/kernel_matrix_testing/types.py b/tasks/kernel_matrix_testing/types.py index af42a6d695810d..6f25f425823a97 100644 --- a/tasks/kernel_matrix_testing/types.py +++ b/tasks/kernel_matrix_testing/types.py @@ -25,10 +25,20 @@ class DependenciesLayout(TypedDict): # noqa: F841 build: Dict[str, DependencyBuild] +class PlatformInfo(TypedDict, total=False): + os_name: str # Official OS name # noqa: F841 + os_version: str # Version # noqa: F841 + image_version: str # Image version # noqa: F841 + kernel: str # Kernel version + os_id: str # Short ID for the OS (e.g., "centos" for CentOS) # noqa: F841 + image: str # Name of the image file + alt_version_names: List[str] # Alternative version names (e.g., "jammy" for Ubuntu 22) # noqa: F841 + + class Platforms(TypedDict): # noqa: F841 url_base: str - x86_64: Dict[str, str] # noqa: F841 - arm64: Dict[str, str] # noqa: F841 + x86_64: Dict[str, PlatformInfo] # noqa: F841 + arm64: Dict[str, PlatformInfo] # noqa: F841 class Disk(TypedDict): diff --git a/tasks/kernel_matrix_testing/vars.py b/tasks/kernel_matrix_testing/vars.py index 10920a98aef38b..222464fa827e85 100644 --- a/tasks/kernel_matrix_testing/vars.py +++ b/tasks/kernel_matrix_testing/vars.py @@ -14,5 +14,6 @@ "arm": "arm64", "aarch64": "arm64", } +arch_ls: list[Arch] = ["x86_64", "arm64"] VMCONFIG = "vmconfig.json" diff --git a/tasks/kernel_matrix_testing/vmconfig.py b/tasks/kernel_matrix_testing/vmconfig.py index 75c90b7ae3dcce..711586f7bc983d 100644 --- a/tasks/kernel_matrix_testing/vmconfig.py +++ b/tasks/kernel_matrix_testing/vmconfig.py @@ -5,6 +5,8 @@ import json import os import platform +import random +from collections import defaultdict from typing import TYPE_CHECKING, Any, List, Optional, Set, Union, cast from urllib.parse import urlparse @@ -14,7 +16,7 @@ from tasks.kernel_matrix_testing.platforms import get_platforms from tasks.kernel_matrix_testing.stacks import check_and_get_stack, create_stack, stack_exists from tasks.kernel_matrix_testing.tool import Exit, ask, info, warn -from tasks.kernel_matrix_testing.vars import VMCONFIG, arch_mapping +from tasks.kernel_matrix_testing.vars import VMCONFIG, arch_ls, arch_mapping if TYPE_CHECKING: from tasks.kernel_matrix_testing.types import ( # noqa: F401 @@ -78,48 +80,9 @@ "4.19", "4.20", ] -distributions = { - # Ubuntu mappings - "ubuntu_16": "ubuntu_16.04", - "ubuntu_18": "ubuntu_18.04", - "ubuntu_20": "ubuntu_20.04", - "ubuntu_22": "ubuntu_22.04", - "ubuntu_23": "ubuntu_23.10", - "xenial": "ubuntu_16.04", - "bionic": "ubuntu_18.04", - "focal": "ubuntu_20.04", - "jammy": "ubuntu_22.04", - "mantic": "ubuntu_23.10", - # Amazon Linux mappings - "amazon_4.14": "amzn_4.14", - "amazon_5.4": "amzn_5.4", - "amazon_5.10": "amzn_5.10", - "amzn_4.14": "amzn_4.14", - "amzn_414": "amzn_4.14", - "amzn_5.4": "amzn_5.4", - "amzn_5.10": "amzn_5.10", - "amzn_2023": "amzn_2023", - "amazon_2023": "amzn_2023", - "al3": "amzn_2023", - "amzn_3": "amzn_2023", - # Fedora mappings - "fedora_37": "fedora_37", - "fedora_38": "fedora_38", - # Debian mappings - "debian_10": "debian_10", - "debian_11": "debian_11", - "debian_12": "debian_12", - # CentOS mappings - "centos_79": "centos_79", - "centos_7": "centos_79", - "centos_8": "centos_8", - # Rocky Linux mappings - "rocky_8.5": "rocky_8.5", - "rocky_9.3": "rocky_9.3", -} - -TICK = "\u2713" -CROSS = "\u2718" + +TICK = "\033[32m\u2713\033[0m" +CROSS = "\033[31m\u2718\033[0m" table = [ ["Image", "x86_64", "arm64"], ["ubuntu-18 (bionic)", TICK, CROSS], @@ -153,22 +116,71 @@ def lte_414(version: str) -> bool: return (int(major) <= 4) and (int(minor) <= 14) -def get_image_list(distro: bool, custom: bool) -> List[List[str]]: - custom_kernels: List[List[str]] = list() - for k in kernels: +def get_image_list(distro: bool, custom: bool) -> list[list[str]]: + headers = [ + "VM name", + "OS Name", + "OS Version", + "Kernel", + "x86_64", + "arm64", + "Alternative names", + "Example VM tags to use with --vms (fuzzy matching)", + ] + custom_kernels: list[list[str]] = list() + for k in sorted(kernels, key=lambda x: tuple(map(int, x.split('.')))): if lte_414(k): - custom_kernels.append([f"custom kernel v{k}", TICK, CROSS]) + custom_kernels.append([f"custom-{k}", "Debian", "Custom", k, TICK, CROSS, "", f"custom-{k}-x86_64"]) else: - custom_kernels.append([f"custom kernel v{k}", TICK, TICK]) - - if (not (distro or custom)) or (distro and custom): - return table + custom_kernels - elif distro: - return table - elif custom: - return custom_kernels - else: - return [] + custom_kernels.append([f"custom-{k}", "Debian", "Custom", k, TICK, TICK, "", f"custom-{k}-x86_64"]) + + distro_kernels: list[list[str]] = list() + platforms = get_platforms() + mappings = get_distribution_mappings() + # Group kernels by name and kernel version, show whether one or two architectures are supported + for arch in arch_ls: + for name, platinfo in platforms[arch].items(): + if isinstance(platinfo, str): + continue # Old format + + # See if we've already added this kernel but for a different architecture. If not, create the entry. + entry = None + for row in distro_kernels: + if row[0] == name and row[3] == platinfo.get('kernel'): + entry = row + break + if entry is None: + names = {k for k, v in mappings.items() if v == name} + # Take two random names for the table so users get an idea of possible mappings + names = random.choices(list(names), k=min(2, len(names))) + + entry = [ + name, + platinfo.get("os_name"), + platinfo.get("os_version"), + platinfo.get("kernel"), + CROSS, + CROSS, + ", ".join(platinfo.get("alt_version_names", [])), + ", ".join(f"distro-{n}-{arch}" for n in names), + ] + distro_kernels.append(entry) + + if arch == "x86_64": + entry[4] = TICK + else: + entry[5] = TICK + + # Sort by name + distro_kernels.sort(key=lambda x: x[0]) + + table = [headers] + if distro: + table += distro_kernels + if custom: + table += custom_kernels + + return table def check_memory_and_vcpus(memory: List[Any], vcpus: List[Any]): @@ -187,8 +199,51 @@ def empty_config(file_path: str): f.write(j) -def list_possible() -> List[str]: - distros = list(distributions.keys()) +def get_distribution_mappings() -> dict[str, str]: + platforms = get_platforms() + distro_mappings: dict[str, str] = dict() + alternative_spellings = {"amzn": ["amazon", "al"]} + mapping_candidates: dict[str, set[str]] = defaultdict( + set + ) # Store here maps that could generate duplicates. Values are the possible targets + + for arch in arch_ls: + for name, platinfo in platforms[arch].items(): + if isinstance(platinfo, str): + continue # Avoid a crash if we have the old format in the platforms file + if name in distro_mappings: + continue # Ignore already existing images (from other arch) + + distro_mappings[name] = name # Direct name + distro_mappings[name.replace('.', '')] = name # Allow name without dots + for alt in platinfo.get("alt_version_names", []): + distro_mappings[alt] = name # Alternative version names map directly to the main name + + os_id = platinfo.get("os_id", "") + version = platinfo.get('version', "") + + if version != "": + if ( + os_id != "" and os_id != name.split('_')[0] + ): # If the os_id is different from the main name, add it too + distro_mappings[f"{os_id}_{version}"] = name + + for alt in alternative_spellings.get(os_id, []): + distro_mappings[f"{alt}_{version}"] = name + + name_no_minor_version = f"{os_id}_{version.split('.')[0]}" + mapping_candidates[name_no_minor_version].add(name) + + # Add candidates that didn't have any duplicates + for name, candidates in mapping_candidates.items(): + if len(candidates) == 1: + distro_mappings[name] = candidates.pop() + + return distro_mappings + + +def list_possible() -> list[str]: + distros = list(get_distribution_mappings().keys()) archs = list(arch_mapping.keys()) archs.append(local_arch) @@ -222,7 +277,7 @@ def normalize_vm_def(possible: List[str], vm: str) -> VMDef: arch = arch_mapping[arch] if recipe == "distro": - version = distributions[version] + version = get_distribution_mappings()[version] elif recipe != "custom": raise Exit(f"Invalid recipe {recipe}") @@ -273,7 +328,10 @@ def get_kernel_config( arch = arch_mapping[platform.machine()] url_base = platforms["url_base"] - kernel_path = platforms[arch][version] + platinfo = platforms[arch][version] + if "image" not in platinfo or "image_version" not in platinfo: + raise Exit(f"image not found in platform information for {version}") + kernel_path = f"{platinfo['image_version']}/{platinfo['image']}" kernel_name = xz_suffix_removed(os.path.basename(kernel_path)) return {"tag": version, "image_source": os.path.join(url_base, kernel_path), "dir": kernel_name} diff --git a/tasks/kmt.py b/tasks/kmt.py index 42d267a0f7064f..b5705e3274e100 100644 --- a/tasks/kmt.py +++ b/tasks/kmt.py @@ -22,8 +22,10 @@ from tasks.kernel_matrix_testing.infra import HostInstance, LibvirtDomain, build_infrastructure from tasks.kernel_matrix_testing.init_kmt import init_kernel_matrix_testing_system from tasks.kernel_matrix_testing.kmt_os import get_kmt_os +from tasks.kernel_matrix_testing.platforms import get_platforms, platforms_file from tasks.kernel_matrix_testing.stacks import check_and_get_stack, ec2_instance_ids from tasks.kernel_matrix_testing.tool import Exit, ask, info, warn +from tasks.kernel_matrix_testing.vars import arch_ls from tasks.libs.common.gitlab import Gitlab, get_gitlab_token from tasks.system_probe import EMBEDDED_SHARE_DIR @@ -247,7 +249,7 @@ def init(ctx: Context, lite=False): @task -def update_resources(ctx: Context, vmconfig_template="system-probe"): +def update_resources(ctx: Context, vmconfig_template="system-probe", all_archs=False): kmt_os = get_kmt_os() warn("Updating resource dependencies will delete all running stacks.") @@ -257,7 +259,7 @@ def update_resources(ctx: Context, vmconfig_template="system-probe"): for stack in glob(f"{kmt_os.stacks_dir}/*"): destroy_stack(ctx, stack=os.path.basename(stack)) - update_rootfs(ctx, kmt_os.rootfs_dir, vmconfig_template) + update_rootfs(ctx, kmt_os.rootfs_dir, vmconfig_template, all_archs=all_archs) @task @@ -771,3 +773,156 @@ def status(ctx: Context, stack: Optional[str] = None, all=False): for line in lines: print(line) print("") + + +@task( + help={ + "version": "The version to update the images to. If not provided, version will not be changed. If 'latest' is provided, the latest version will be used.", + "update-only-matching": "Only update the platform info for images that match the given regex", + "exclude-matching": "Exclude images that match the given regex", + } +) +def update_platform_info( + ctx: Context, + version: str | None = None, + update_only_matching: str | None = None, + exclude_matching: str | None = None, +): + """Generate a JSON file with platform information for all the images + found in the KMT S3 bucket. + """ + res = ctx.run( + "aws-vault exec sso-staging-engineering -- aws s3 ls --recursive s3://dd-agent-omnibus/kernel-version-testing/rootfs", + warn=True, + ) + if res is None or not res.ok: + raise Exit("Cannot list bucket contents") + + objects = [line.split()[-1] for line in res.stdout.splitlines()] + objects_by_version: dict[str, list[str]] = defaultdict(list) + + for obj in objects: + v = "/".join(obj.split("/")[2:-1]) + if v != "": + objects_by_version[v].append(obj) + + if version is None: + master_versions = [v for v in objects_by_version if re.match(r"^20[0-9]{6}_[0-9a-f]+$", v)] + if len(master_versions) == 0: + raise Exit("No master versions available") + + version = sorted(master_versions)[-1] + info(f"[+] detected {version} as latest version from master branch") + + if version not in objects_by_version: + raise Exit(f"Version {version} not found in S3 bucket, cannot update") + + manifests = [obj for obj in objects_by_version[version] if obj.endswith(".manifest")] + platforms = get_platforms() + + with tempfile.TemporaryDirectory() as tmpdir: + for manifest in manifests: + info(f"[+] Processing manifest {manifest}") + ctx.run(f"aws-vault exec sso-staging-engineering -- aws s3 cp s3://dd-agent-omnibus/{manifest} {tmpdir}") + with open(f"{tmpdir}/{os.path.basename(manifest)}") as f: + options = f.readlines() + keyvals = {line.split("=")[0]: line.split("=")[1].strip().strip('"') for line in options} + + try: + arch = arch_mapping[keyvals['ARCH']] + image_name = keyvals['IMAGE_NAME'] + image_filename = keyvals['IMAGE_FILENAME'] + except KeyError: + raise Exit(f"[!] Invalid manifest {manifest}") + + if arch not in platforms: + warn(f"[!] Unsupported architecture {arch}, skipping") + continue + + if update_only_matching is not None and re.search(update_only_matching, image_name) is None: + warn(f"[!] Image {image_name} does not match the filter, skipping") + continue + + if exclude_matching is not None and re.search(exclude_matching, image_name) is not None: + warn(f"[!] Image {image_name} matches the exclude filter, skipping") + continue + + manifest_to_platinfo_keys = { + 'NAME': 'os_name', + 'ID': 'os_id', + 'KERNEL_VERSION': 'kernel', + 'VERSION_ID': 'os_version', + } + + if image_name not in platforms[arch]: + platforms[arch][image_name] = {} + + for mkey, pkey in manifest_to_platinfo_keys.items(): + if mkey in keyvals: + platforms[arch][image_name][pkey] = keyvals[mkey] + + platforms[arch][image_name]['image'] = image_filename + ".xz" + platforms[arch][image_name]['image_version'] = version + + if 'VERSION_CODENAME' in keyvals: + altname = keyvals['VERSION_CODENAME'] + # Do not modify existing altnames + altnames = platforms[arch][image_name].get('alt_version_names', []) + if altname not in altnames: + altnames.append(altname) + + platforms[arch][image_name]['alt_version_names'] = altnames + + info(f"[+] Writing output to {platforms_file}...") + + # Do validation of the platforms dict, check that there are no outdated versions + for arch in arch_ls: + for image_name, platinfo in platforms[arch].items(): + if update_only_matching is not None and re.search(update_only_matching, image_name) is None: + continue # Only validate those that match + + if exclude_matching is not None and re.search(exclude_matching, image_name) is not None: + continue + + version_from_file = platinfo.get('image_version') + if version_from_file != version: + warn( + f"[!] Image {image_name} ({arch}) has version {version_from_file} but we are updating to {version}, manifest file may be missing?" + ) + + with open(platforms_file, "w") as f: + json.dump(platforms, f, indent=2) + + +@task +def validate_platform_info(ctx: Context): + """Validate the platform info file for correctness, ensuring that all images can be found""" + platforms = get_platforms() + errors: set[str] = set() + + for arch in arch_ls: + for image_name, platinfo in platforms[arch].items(): + image = platinfo.get('image') + if image is None: + warn(f"[!] {image_name} does not have an image filename") + errors.add(image_name) + continue + + version = platinfo.get('image_version') + if version is None: + warn(f"[!] {image_name} does not have an image version") + errors.add(image_name) + continue + + remote = f"{platforms['url_base']}/{version}/{image}" + res = ctx.run(f"curl -s -I {remote}") + if res is None or not res.ok: + warn(f"[!] {image_name}: {image} for version {version} not found at {remote}") + errors.add(image_name) + else: + info(f"[+] {image_name}: {image} for version {version} found at {remote}") + + if len(errors) == 0: + info("[+] Platform info file is valid") + else: + raise Exit(f"[!] Found {len(errors)} errors in the platform info file. Images failed: {', '.join(errors)}") diff --git a/test/new-e2e/system-probe/config/platforms.json b/test/new-e2e/system-probe/config/platforms.json index d6069e0a88e808..84dc0ba917ae40 100644 --- a/test/new-e2e/system-probe/config/platforms.json +++ b/test/new-e2e/system-probe/config/platforms.json @@ -1,45 +1,372 @@ { - "url_base": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs", - "x86_64": { - "ubuntu_16.04": "master/ubuntu-16-x86_64.qcow2.xz", - "ubuntu_18.04": "master/ubuntu-18-x86_64.qcow2.xz", - "ubuntu_20.04": "master/ubuntu-20-x86_64.qcow2.xz", - "ubuntu_22.04": "master/ubuntu-22-x86_64.qcow2.xz", - "ubuntu_23.10": "master/ubuntu-23-x86_64.qcow2.xz", - "amzn_4.14": "master/amzn2-kvm-2.0-x86_64-4.14.qcow2.xz", - "amzn_5.4": "master/amzn2-kvm-2.0-x86_64-5.4.qcow2.xz", - "amzn_5.10": "master/amzn2-kvm-2.0-x86_64-5.10.qcow2.xz", - "fedora_37": "master/Fedora-Cloud-Base-37.x86_64.qcow2.xz", - "fedora_38": "master/Fedora-Cloud-Base-38.x86_64.qcow2.xz", - "debian_10": "master/debian-10-generic-x86_64.qcow2.xz", - "debian_11": "master/debian-11-generic-x86_64.qcow2.xz", - "debian_12": "master/debian-12-generic-x86_64.qcow2.xz", - "centos_79": "master/centos-7.9-x86_64.qcow2.xz", - "centos_8": "master/centos-8-x86_64.qcow2.xz", - "oracle_7.9": "master/oracle-7.9-x86_64.qcow.xz", - "oracle_8.9": "master/oracle-8.9-x86_64.qcow.xz", - "oracle_9.3": "master/oracle-9.3-x86_64.qcow.xz", - "rocky_8.5": "master/rocky-8.5-x86_64.qcow2.xz", - "rocky_9.3": "master/rocky-9.3-x86_64.qcow2.xz" - }, - "arm64": { - "ubuntu_18.04": "master/ubuntu-18-arm64.qcow2.xz", - "ubuntu_20.04": "master/ubuntu-20-arm64.qcow2.xz", - "ubuntu_22.04": "master/ubuntu-22-arm64.qcow2.xz", - "ubuntu_23.10": "master/ubuntu-23-arm64.qcow2.xz", - "amzn_4.14": "master/amzn2-kvm-2.0-arm64-4.14.qcow2.xz", - "amzn_5.4": "master/amzn2-kvm-2.0-arm64-5.4.qcow2.xz", - "amzn_5.10": "master/amzn2-kvm-2.0-arm64-5.10.qcow2.xz", - "fedora_37": "master/Fedora-Cloud-Base-37.arm64.qcow2.xz", - "fedora_38": "master/Fedora-Cloud-Base-38.arm64.qcow2.xz", - "debian_10": "master/debian-10-generic-arm64.qcow2.xz", - "debian_11": "master/debian-11-generic-arm64.qcow2.xz", - "debian_12": "master/debian-12-generic-arm64.qcow2.xz", - "centos_79": "master/centos-7.9-arm64.qcow2.xz", - "centos_8": "master/centos-8-arm64.qcow2.xz", - "oracle_8.9": "master/oracle-8.9-arm64.qcow.xz", - "oracle_9.3": "master/oracle-9.3-arm64.qcow.xz", - "rocky_8.5": "master/rocky-8.5-arm64.qcow2.xz", - "rocky_9.3": "master/rocky-9.3-arm64.qcow2.xz" + "url_base": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs", + "x86_64": { + "fedora_37": { + "os_name": "Fedora Linux", + "os_id": "fedora", + "kernel": "6.0.7-301.fc37", + "os_version": "37", + "image": "Fedora-Cloud-Base-37.x86_64.qcow2.xz", + "image_version": "20240424_95349298" + }, + "fedora_38": { + "os_name": "Fedora Linux", + "os_id": "fedora", + "kernel": "6.2.9-300.fc38", + "os_version": "38", + "image": "Fedora-Cloud-Base-38.x86_64.qcow2.xz", + "image_version": "20240424_95349298" + }, + "amazon_4.14": { + "os_name": "Amazon Linux", + "os_id": "amzn", + "kernel": "4.14.314-237.533.amzn2", + "os_version": "2", + "image": "amzn2-x86_64-4.14.qcow2.xz", + "image_version": "20240424_95349298" + }, + "amazon_5.10": { + "os_name": "Amazon Linux", + "os_id": "amzn", + "kernel": "5.10.214-202.855.amzn2", + "os_version": "2", + "image": "amzn2-x86_64-5.10.qcow2.xz", + "image_version": "20240424_95349298" + }, + "amazon_5.4": { + "os_name": "Amazon Linux", + "os_id": "amzn", + "kernel": "5.4.273-186.370.amzn2", + "os_version": "2", + "image": "amzn2-x86_64-5.4.qcow2.xz", + "image_version": "20240424_95349298" + }, + "amazon_2023": { + "os_name": "Amazon Linux", + "os_id": "amzn", + "kernel": "6.1.77-99.164.amzn2023", + "os_version": "2023", + "image": "amzn2023-amd64-2023.qcow2.xz", + "image_version": "20240424_95349298" + }, + "centos_7.9": { + "os_name": "CentOS Linux", + "os_id": "centos", + "kernel": "3.10.0-1160.80.1.el7", + "os_version": "7.9.2009", + "image": "centos-7.9-x86_64.qcow2.xz", + "image_version": "20240424_95349298" + }, + "centos_8": { + "os_name": "CentOS Stream", + "os_id": "centos", + "kernel": "4.18.0-545.el8", + "os_version": "8", + "image": "centos-8-x86_64.qcow2.xz", + "image_version": "20240424_95349298" + }, + "debian_10": { + "os_name": "Debian GNU/Linux", + "os_id": "debian", + "kernel": "4.19.0-26", + "os_version": "10", + "image": "debian-10-generic-x86_64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "buster" + ] + }, + "debian_11": { + "os_name": "Debian GNU/Linux", + "os_id": "debian", + "kernel": "5.10.0-28", + "os_version": "11", + "image": "debian-11-generic-x86_64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "bullseye" + ] + }, + "debian_12": { + "os_name": "Debian GNU/Linux", + "os_id": "debian", + "kernel": "6.1.0-18", + "os_version": "12", + "image": "debian-12-generic-x86_64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "bookworm" + ] + }, + "oracle_7.9": { + "os_name": "Oracle Linux Server", + "os_id": "ol", + "kernel": "3.10.0-1160.105.1.0.1.el7", + "os_version": "7.9", + "image": "oracle-7.9-x86_64.qcow.xz", + "image_version": "20240424_95349298" + }, + "oracle_8.9": { + "os_name": "Oracle Linux Server", + "os_id": "ol", + "kernel": "5.15.0-200.131.27.el8uek", + "os_version": "8.9", + "image": "oracle-8.9-x86_64.qcow.xz", + "image_version": "20240424_95349298" + }, + "oracle_9.3": { + "os_name": "Oracle Linux Server", + "os_id": "ol", + "kernel": "5.15.0-200.131.27.el9uek", + "os_version": "9.3", + "image": "oracle-9.3-x86_64.qcow.xz", + "image_version": "20240424_95349298" + }, + "rocky_8.5": { + "os_name": "Rocky Linux", + "os_id": "rocky", + "kernel": "4.18.0-348.el8.0.2", + "os_version": "8.5", + "image": "rocky-8.5-x86_64.qcow2.xz", + "image_version": "20240424_95349298" + }, + "rocky_9.3": { + "os_name": "Rocky Linux", + "os_id": "rocky", + "kernel": "5.14.0-362.8.1.el9_3", + "os_version": "9.3", + "image": "rocky-9.3-x86_64.qcow2.xz", + "image_version": "20240424_95349298" + }, + "ubuntu_16.04": { + "os_name": "Ubuntu", + "os_id": "ubuntu", + "kernel": "4.4.0-210-generic", + "os_version": "16.04", + "image": "ubuntu-16.04-x86_64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "xenial" + ] + }, + "ubuntu_18.04": { + "os_name": "Ubuntu", + "os_id": "ubuntu", + "kernel": "4.18.0-25-generic", + "os_version": "18.04", + "image": "ubuntu-18.04-x86_64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "bionic" + ] + }, + "ubuntu_20.04": { + "os_name": "Ubuntu", + "os_id": "ubuntu", + "kernel": "5.4.0-167-generic", + "os_version": "20.04", + "image": "ubuntu-20.04-x86_64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "focal" + ] + }, + "ubuntu_22.04": { + "os_name": "Ubuntu", + "os_id": "ubuntu", + "kernel": "5.15.0-91-generic", + "os_version": "22.04", + "image": "ubuntu-22.04-x86_64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "jammy" + ] + }, + "ubuntu_23.10": { + "os_name": "Ubuntu", + "os_id": "ubuntu", + "kernel": "6.5.0-9-generic", + "os_version": "23.10", + "image": "ubuntu-23.10-x86_64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "mantic" + ] + } + }, + "arm64": { + "fedora_37": { + "os_name": "Fedora Linux", + "os_id": "fedora", + "kernel": "6.0.7-301.fc37", + "os_version": "37", + "image": "Fedora-Cloud-Base-37.arm64.qcow2.xz", + "image_version": "20240424_95349298" + }, + "fedora_38": { + "os_name": "Fedora Linux", + "os_id": "fedora", + "kernel": "6.2.9-300.fc38", + "os_version": "38", + "image": "Fedora-Cloud-Base-38.arm64.qcow2.xz", + "image_version": "20240424_95349298" + }, + "amazon_4.14": { + "os_name": "Amazon Linux", + "os_id": "amzn", + "kernel": "4.14.314-237.533.amzn2", + "os_version": "2", + "image": "amzn2-arm64-4.14.qcow2.xz", + "image_version": "20240424_95349298" + }, + "amazon_5.4": { + "os_name": "Amazon Linux", + "os_id": "amzn", + "kernel": "5.4.273-186.370.amzn2", + "os_version": "2", + "image": "amzn2-arm64-5.4.qcow2.xz", + "image_version": "20240424_95349298" + }, + "centos_7.9": { + "os_name": "CentOS Linux", + "os_id": "centos", + "kernel": "4.18.0-348.20.1.el7", + "os_version": "7.9.2009", + "image": "centos-7.9-arm64.qcow2.xz", + "image_version": "20240424_95349298" + }, + "centos_8": { + "os_name": "CentOS Stream", + "os_id": "centos", + "kernel": "4.18.0-545.el8", + "os_version": "8", + "image": "centos-8-arm64.qcow2.xz", + "image_version": "20240424_95349298" + }, + "debian_10": { + "os_name": "Debian GNU/Linux", + "os_id": "debian", + "kernel": "4.19.0-26", + "os_version": "10", + "image": "debian-10-generic-arm64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "buster" + ] + }, + "debian_11": { + "os_name": "Debian GNU/Linux", + "os_id": "debian", + "kernel": "5.10.0-28", + "os_version": "11", + "image": "debian-11-generic-arm64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "bullseye" + ] + }, + "debian_12": { + "os_name": "Debian GNU/Linux", + "os_id": "debian", + "kernel": "6.1.0-18", + "os_version": "12", + "image": "debian-12-generic-arm64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "bookworm" + ] + }, + "oracle_8.9": { + "os_name": "Oracle Linux Server", + "os_id": "ol", + "kernel": "5.15.0-200.131.27.el8uek", + "os_version": "8.9", + "image": "oracle-8.9-arm64.qcow.xz", + "image_version": "20240424_95349298" + }, + "oracle_9.3": { + "os_name": "Oracle Linux Server", + "os_id": "ol", + "kernel": "5.15.0-200.131.27.el9uek", + "os_version": "9.3", + "image": "oracle-9.3-arm64.qcow.xz", + "image_version": "20240424_95349298" + }, + "rocky_8.5": { + "os_name": "Rocky Linux", + "os_id": "rocky", + "kernel": "4.18.0-348.el8.0.2", + "os_version": "8.5", + "image": "rocky-8.5-arm64.qcow2.xz", + "image_version": "20240424_95349298" + }, + "rocky_9.3": { + "os_name": "Rocky Linux", + "os_id": "rocky", + "kernel": "5.14.0-362.8.1.el9_3", + "os_version": "9.3", + "image": "rocky-9.3-arm64.qcow2.xz", + "image_version": "20240424_95349298" + }, + "amazon_5.10": { + "os_name": "Amazon Linux", + "os_id": "amzn", + "kernel": "5.10.214-202.855.amzn2", + "os_version": "2", + "image": "amzn2-arm64-5.10.qcow2.xz", + "image_version": "20240424_95349298" + }, + "amazon_2023": { + "os_name": "Amazon Linux", + "os_id": "amzn", + "kernel": "6.1.77-99.164.amzn2023", + "os_version": "2023", + "image": "amzn2023-arm64-2023.qcow2.xz", + "image_version": "20240424_95349298" + }, + "ubuntu_18.04": { + "os_name": "Ubuntu", + "os_id": "ubuntu", + "kernel": "4.18.0-25-generic", + "os_version": "18.04", + "image": "ubuntu-18.04-arm64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "bionic" + ] + }, + "ubuntu_20.04": { + "os_name": "Ubuntu", + "os_id": "ubuntu", + "kernel": "5.4.0-167-generic", + "os_version": "20.04", + "image": "ubuntu-20.04-arm64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "focal" + ] + }, + "ubuntu_22.04": { + "os_name": "Ubuntu", + "os_id": "ubuntu", + "kernel": "5.15.0-91-generic", + "os_version": "22.04", + "image": "ubuntu-22.04-arm64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "jammy" + ] + }, + "ubuntu_23.10": { + "os_name": "Ubuntu", + "os_id": "ubuntu", + "kernel": "6.5.0-9-generic", + "os_version": "23.10", + "image": "ubuntu-23.10-arm64.qcow2.xz", + "image_version": "20240424_95349298", + "alt_version_names": [ + "mantic" + ] } + } } From 88800602187e34b89a5db2da1abb811a2b3520a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20Julia=CC=81n?= Date: Tue, 3 Dec 2024 13:32:31 +0100 Subject: [PATCH 02/12] Update platforms --- .../system-probe/config/platforms.json | 108 +++++++++--------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/test/new-e2e/system-probe/config/platforms.json b/test/new-e2e/system-probe/config/platforms.json index 84dc0ba917ae40..d440af9e51e682 100644 --- a/test/new-e2e/system-probe/config/platforms.json +++ b/test/new-e2e/system-probe/config/platforms.json @@ -7,7 +7,7 @@ "kernel": "6.0.7-301.fc37", "os_version": "37", "image": "Fedora-Cloud-Base-37.x86_64.qcow2.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "fedora_38": { "os_name": "Fedora Linux", @@ -15,7 +15,7 @@ "kernel": "6.2.9-300.fc38", "os_version": "38", "image": "Fedora-Cloud-Base-38.x86_64.qcow2.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "amazon_4.14": { "os_name": "Amazon Linux", @@ -23,23 +23,23 @@ "kernel": "4.14.314-237.533.amzn2", "os_version": "2", "image": "amzn2-x86_64-4.14.qcow2.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "amazon_5.10": { "os_name": "Amazon Linux", "os_id": "amzn", - "kernel": "5.10.214-202.855.amzn2", + "kernel": "5.10.226-214.880.amzn2", "os_version": "2", "image": "amzn2-x86_64-5.10.qcow2.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "amazon_5.4": { "os_name": "Amazon Linux", "os_id": "amzn", - "kernel": "5.4.273-186.370.amzn2", + "kernel": "5.4.284-196.381.amzn2", "os_version": "2", "image": "amzn2-x86_64-5.4.qcow2.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "amazon_2023": { "os_name": "Amazon Linux", @@ -47,7 +47,7 @@ "kernel": "6.1.77-99.164.amzn2023", "os_version": "2023", "image": "amzn2023-amd64-2023.qcow2.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "centos_7.9": { "os_name": "CentOS Linux", @@ -55,23 +55,23 @@ "kernel": "3.10.0-1160.80.1.el7", "os_version": "7.9.2009", "image": "centos-7.9-x86_64.qcow2.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "centos_8": { "os_name": "CentOS Stream", "os_id": "centos", - "kernel": "4.18.0-545.el8", + "kernel": "4.18.0-552.3.1.el8", "os_version": "8", "image": "centos-8-x86_64.qcow2.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "debian_10": { "os_name": "Debian GNU/Linux", "os_id": "debian", - "kernel": "4.19.0-26", + "kernel": "4.19.0-27", "os_version": "10", "image": "debian-10-generic-x86_64.qcow2.xz", - "image_version": "20240424_95349298", + "image_version": "20241015_1cc052c3", "alt_version_names": [ "buster" ] @@ -79,10 +79,10 @@ "debian_11": { "os_name": "Debian GNU/Linux", "os_id": "debian", - "kernel": "5.10.0-28", + "kernel": "5.10.0-32", "os_version": "11", "image": "debian-11-generic-x86_64.qcow2.xz", - "image_version": "20240424_95349298", + "image_version": "20241015_1cc052c3", "alt_version_names": [ "bullseye" ] @@ -90,10 +90,10 @@ "debian_12": { "os_name": "Debian GNU/Linux", "os_id": "debian", - "kernel": "6.1.0-18", + "kernel": "6.1.0-25", "os_version": "12", "image": "debian-12-generic-x86_64.qcow2.xz", - "image_version": "20240424_95349298", + "image_version": "20241015_1cc052c3", "alt_version_names": [ "bookworm" ] @@ -104,7 +104,7 @@ "kernel": "3.10.0-1160.105.1.0.1.el7", "os_version": "7.9", "image": "oracle-7.9-x86_64.qcow.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "oracle_8.9": { "os_name": "Oracle Linux Server", @@ -112,7 +112,7 @@ "kernel": "5.15.0-200.131.27.el8uek", "os_version": "8.9", "image": "oracle-8.9-x86_64.qcow.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "oracle_9.3": { "os_name": "Oracle Linux Server", @@ -120,7 +120,7 @@ "kernel": "5.15.0-200.131.27.el9uek", "os_version": "9.3", "image": "oracle-9.3-x86_64.qcow.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "rocky_8.5": { "os_name": "Rocky Linux", @@ -128,7 +128,7 @@ "kernel": "4.18.0-348.el8.0.2", "os_version": "8.5", "image": "rocky-8.5-x86_64.qcow2.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "rocky_9.3": { "os_name": "Rocky Linux", @@ -136,7 +136,7 @@ "kernel": "5.14.0-362.8.1.el9_3", "os_version": "9.3", "image": "rocky-9.3-x86_64.qcow2.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "ubuntu_16.04": { "os_name": "Ubuntu", @@ -144,7 +144,7 @@ "kernel": "4.4.0-210-generic", "os_version": "16.04", "image": "ubuntu-16.04-x86_64.qcow2.xz", - "image_version": "20240424_95349298", + "image_version": "20241015_1cc052c3", "alt_version_names": [ "xenial" ] @@ -155,7 +155,7 @@ "kernel": "4.18.0-25-generic", "os_version": "18.04", "image": "ubuntu-18.04-x86_64.qcow2.xz", - "image_version": "20240424_95349298", + "image_version": "20241015_1cc052c3", "alt_version_names": [ "bionic" ] @@ -166,7 +166,7 @@ "kernel": "5.4.0-167-generic", "os_version": "20.04", "image": "ubuntu-20.04-x86_64.qcow2.xz", - "image_version": "20240424_95349298", + "image_version": "20241015_1cc052c3", "alt_version_names": [ "focal" ] @@ -177,7 +177,7 @@ "kernel": "5.15.0-91-generic", "os_version": "22.04", "image": "ubuntu-22.04-x86_64.qcow2.xz", - "image_version": "20240424_95349298", + "image_version": "20241015_1cc052c3", "alt_version_names": [ "jammy" ] @@ -185,10 +185,10 @@ "ubuntu_23.10": { "os_name": "Ubuntu", "os_id": "ubuntu", - "kernel": "6.5.0-9-generic", + "kernel": "6.5.0-44-generic", "os_version": "23.10", "image": "ubuntu-23.10-x86_64.qcow2.xz", - "image_version": "20240424_95349298", + "image_version": "20241015_1cc052c3", "alt_version_names": [ "mantic" ] @@ -201,7 +201,7 @@ "kernel": "6.0.7-301.fc37", "os_version": "37", "image": "Fedora-Cloud-Base-37.arm64.qcow2.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "fedora_38": { "os_name": "Fedora Linux", @@ -209,7 +209,7 @@ "kernel": "6.2.9-300.fc38", "os_version": "38", "image": "Fedora-Cloud-Base-38.arm64.qcow2.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "amazon_4.14": { "os_name": "Amazon Linux", @@ -217,15 +217,15 @@ "kernel": "4.14.314-237.533.amzn2", "os_version": "2", "image": "amzn2-arm64-4.14.qcow2.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "amazon_5.4": { "os_name": "Amazon Linux", "os_id": "amzn", - "kernel": "5.4.273-186.370.amzn2", + "kernel": "5.4.284-196.381.amzn2", "os_version": "2", "image": "amzn2-arm64-5.4.qcow2.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "centos_7.9": { "os_name": "CentOS Linux", @@ -233,23 +233,23 @@ "kernel": "4.18.0-348.20.1.el7", "os_version": "7.9.2009", "image": "centos-7.9-arm64.qcow2.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "centos_8": { "os_name": "CentOS Stream", "os_id": "centos", - "kernel": "4.18.0-545.el8", + "kernel": "4.18.0-552.3.1.el8", "os_version": "8", "image": "centos-8-arm64.qcow2.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "debian_10": { "os_name": "Debian GNU/Linux", "os_id": "debian", - "kernel": "4.19.0-26", + "kernel": "4.19.0-27", "os_version": "10", "image": "debian-10-generic-arm64.qcow2.xz", - "image_version": "20240424_95349298", + "image_version": "20241015_1cc052c3", "alt_version_names": [ "buster" ] @@ -257,10 +257,10 @@ "debian_11": { "os_name": "Debian GNU/Linux", "os_id": "debian", - "kernel": "5.10.0-28", + "kernel": "5.10.0-32", "os_version": "11", "image": "debian-11-generic-arm64.qcow2.xz", - "image_version": "20240424_95349298", + "image_version": "20241015_1cc052c3", "alt_version_names": [ "bullseye" ] @@ -268,10 +268,10 @@ "debian_12": { "os_name": "Debian GNU/Linux", "os_id": "debian", - "kernel": "6.1.0-18", + "kernel": "6.1.0-25", "os_version": "12", "image": "debian-12-generic-arm64.qcow2.xz", - "image_version": "20240424_95349298", + "image_version": "20241015_1cc052c3", "alt_version_names": [ "bookworm" ] @@ -282,7 +282,7 @@ "kernel": "5.15.0-200.131.27.el8uek", "os_version": "8.9", "image": "oracle-8.9-arm64.qcow.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "oracle_9.3": { "os_name": "Oracle Linux Server", @@ -290,7 +290,7 @@ "kernel": "5.15.0-200.131.27.el9uek", "os_version": "9.3", "image": "oracle-9.3-arm64.qcow.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "rocky_8.5": { "os_name": "Rocky Linux", @@ -298,7 +298,7 @@ "kernel": "4.18.0-348.el8.0.2", "os_version": "8.5", "image": "rocky-8.5-arm64.qcow2.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "rocky_9.3": { "os_name": "Rocky Linux", @@ -306,15 +306,15 @@ "kernel": "5.14.0-362.8.1.el9_3", "os_version": "9.3", "image": "rocky-9.3-arm64.qcow2.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "amazon_5.10": { "os_name": "Amazon Linux", "os_id": "amzn", - "kernel": "5.10.214-202.855.amzn2", + "kernel": "5.10.226-214.880.amzn2", "os_version": "2", "image": "amzn2-arm64-5.10.qcow2.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "amazon_2023": { "os_name": "Amazon Linux", @@ -322,7 +322,7 @@ "kernel": "6.1.77-99.164.amzn2023", "os_version": "2023", "image": "amzn2023-arm64-2023.qcow2.xz", - "image_version": "20240424_95349298" + "image_version": "20241015_1cc052c3" }, "ubuntu_18.04": { "os_name": "Ubuntu", @@ -330,7 +330,7 @@ "kernel": "4.18.0-25-generic", "os_version": "18.04", "image": "ubuntu-18.04-arm64.qcow2.xz", - "image_version": "20240424_95349298", + "image_version": "20241015_1cc052c3", "alt_version_names": [ "bionic" ] @@ -341,7 +341,7 @@ "kernel": "5.4.0-167-generic", "os_version": "20.04", "image": "ubuntu-20.04-arm64.qcow2.xz", - "image_version": "20240424_95349298", + "image_version": "20241015_1cc052c3", "alt_version_names": [ "focal" ] @@ -352,7 +352,7 @@ "kernel": "5.15.0-91-generic", "os_version": "22.04", "image": "ubuntu-22.04-arm64.qcow2.xz", - "image_version": "20240424_95349298", + "image_version": "20241015_1cc052c3", "alt_version_names": [ "jammy" ] @@ -360,10 +360,10 @@ "ubuntu_23.10": { "os_name": "Ubuntu", "os_id": "ubuntu", - "kernel": "6.5.0-9-generic", + "kernel": "6.5.0-44-generic", "os_version": "23.10", "image": "ubuntu-23.10-arm64.qcow2.xz", - "image_version": "20240424_95349298", + "image_version": "20241015_1cc052c3", "alt_version_names": [ "mantic" ] From f0fb75f2ba3639b0347ffa9f016715121d2063e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20Juli=C3=A1n?= Date: Fri, 19 Jul 2024 14:22:27 +0200 Subject: [PATCH 03/12] [EBPF] Do not send API key to KMT microVMs (#27635) Co-authored-by: agent-platform-auto-pr[bot] <153269286+agent-platform-auto-pr[bot]@users.noreply.github.com> --- .gitlab/kernel_matrix_testing/common.yml | 6 +++--- test/new-e2e/system-probe/connector/main.go | 10 ++++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.gitlab/kernel_matrix_testing/common.yml b/.gitlab/kernel_matrix_testing/common.yml index 8581599dba23d7..e84dd76a416ad0 100644 --- a/.gitlab/kernel_matrix_testing/common.yml +++ b/.gitlab/kernel_matrix_testing/common.yml @@ -216,9 +216,9 @@ # ssh into each micro-vm and run initialization script. This script will also run the tests. - scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt" - ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt ${MICRO_VM_IP}:/job_env.txt" - - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file /home/kernel-version-testing/ddvm_rsa -vm-cmd '/root/fetch_dependencies.sh ${ARCH} && /opt/kernel-version-testing/micro-vm-init.sh -retry ${RETRY} -test-root /system-probe-tests -packages-run-config /${TEST_SET}.json'" - - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE -vm-cmd "${NESTED_VM_CMD}" - - ssh metal_instance "ssh ${MICRO_VM_IP} '/test-json-review'" + - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file /home/kernel-version-testing/ddvm_rsa -vm-cmd 'CI=true /root/fetch_dependencies.sh ${ARCH} && /opt/kernel-version-testing/micro-vm-init.sh -retry ${RETRY} -test-root /opt/${TEST_COMPONENT}-tests -packages-run-config /${TEST_SET}.json'" + - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE -vm-cmd "${NESTED_VM_CMD}" -send-env-vars DD_API_KEY # Allow DD_API_KEY to be passed to the metal instance, so we can use it to send metrics from the connector. + - ssh metal_instance "ssh ${MICRO_VM_IP} /test-json-review" artifacts: expire_in: 2 weeks when: always diff --git a/test/new-e2e/system-probe/connector/main.go b/test/new-e2e/system-probe/connector/main.go index 471be88f91f405..dbb6e097ca6035 100644 --- a/test/new-e2e/system-probe/connector/main.go +++ b/test/new-e2e/system-probe/connector/main.go @@ -17,6 +17,7 @@ import ( "net" "os" "os/exec" + "strings" "time" "github.com/DataDog/datadog-api-client-go/v2/api/datadog" @@ -44,6 +45,7 @@ type args struct { serverKeepAliveMaxCount int sshFilePath string vmCommand string + sendEnvVars []string } func readArgs() *args { @@ -54,6 +56,7 @@ func readArgs() *args { serverAliveCountPtr := flag.Int("server-alive-count", 560, "Maximum keep alive messages to send before disconnecting upon no reply") sshFilePathPtr := flag.String("ssh-file", "", "Path to private ssh key") vmCmd := flag.String("vm-cmd", "", "command to run on VM") + sendEnvVars := flag.String("send-env-vars", "", "Comma-separated list of environment variables to send through the connection") flag.Parse() @@ -65,6 +68,7 @@ func readArgs() *args { serverKeepAliveMaxCount: *serverAliveCountPtr, sshFilePath: *sshFilePathPtr, vmCommand: *vmCmd, + sendEnvVars: strings.Split(*sendEnvVars, ","), } } @@ -151,8 +155,10 @@ func run() (err error) { return fmt.Errorf("connect: %s", err) } - if val := os.Getenv("DD_API_KEY"); val != "" { - cmd.Env = append(cmd.Env, fmt.Sprintf("DD_API_KEY=%s", val)) + for _, envVar := range args.sendEnvVars { + if val := os.Getenv(envVar); val != "" { + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", envVar, val)) + } } cmd.Command = args.vmCommand From 0dc4b4c4159e74c18cf1fa40738746f09da9248f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20Julia=CC=81n?= Date: Tue, 3 Dec 2024 15:34:38 +0100 Subject: [PATCH 04/12] Ensure sysprobe jobs are cleaned up --- .gitlab/kernel_matrix_testing/system_probe.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab/kernel_matrix_testing/system_probe.yml b/.gitlab/kernel_matrix_testing/system_probe.yml index 2f8f570842649e..8054a73f90c684 100644 --- a/.gitlab/kernel_matrix_testing/system_probe.yml +++ b/.gitlab/kernel_matrix_testing/system_probe.yml @@ -282,6 +282,7 @@ kmt_run_sysprobe_tests_arm64: TEST_COMPONENT: system-probe kmt_sysprobe_cleanup_arm64: + when: always extends: - .kmt_sysprobe_cleanup needs: @@ -292,6 +293,7 @@ kmt_sysprobe_cleanup_arm64: INSTANCE_TYPE: "m6gd.metal" kmt_sysprobe_cleanup_x64: + when: always extends: - .kmt_sysprobe_cleanup needs: From 7260fcf02eb4592cd5d9a0f22aef337dd417c572 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20Julia=CC=81n?= Date: Wed, 4 Dec 2024 10:06:05 +0100 Subject: [PATCH 05/12] Fix test root --- .gitlab/kernel_matrix_testing/common.yml | 2 +- test/new-e2e/system-probe/test/micro-vm-init.sh | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitlab/kernel_matrix_testing/common.yml b/.gitlab/kernel_matrix_testing/common.yml index e84dd76a416ad0..0050ccfb7b84e7 100644 --- a/.gitlab/kernel_matrix_testing/common.yml +++ b/.gitlab/kernel_matrix_testing/common.yml @@ -216,7 +216,7 @@ # ssh into each micro-vm and run initialization script. This script will also run the tests. - scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt" - ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt ${MICRO_VM_IP}:/job_env.txt" - - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file /home/kernel-version-testing/ddvm_rsa -vm-cmd 'CI=true /root/fetch_dependencies.sh ${ARCH} && /opt/kernel-version-testing/micro-vm-init.sh -retry ${RETRY} -test-root /opt/${TEST_COMPONENT}-tests -packages-run-config /${TEST_SET}.json'" + - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file /home/kernel-version-testing/ddvm_rsa -vm-cmd 'CI=true /root/fetch_dependencies.sh ${ARCH} && /opt/kernel-version-testing/micro-vm-init.sh -retry ${RETRY} -test-root /opt/kmt-ramfs/${TEST_COMPONENT}-tests -packages-run-config /${TEST_SET}.json'" - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE -vm-cmd "${NESTED_VM_CMD}" -send-env-vars DD_API_KEY # Allow DD_API_KEY to be passed to the metal instance, so we can use it to send metrics from the connector. - ssh metal_instance "ssh ${MICRO_VM_IP} /test-json-review" artifacts: diff --git a/test/new-e2e/system-probe/test/micro-vm-init.sh b/test/new-e2e/system-probe/test/micro-vm-init.sh index 3762ab87d1fd30..97d5769a7ab209 100755 --- a/test/new-e2e/system-probe/test/micro-vm-init.sh +++ b/test/new-e2e/system-probe/test/micro-vm-init.sh @@ -13,6 +13,11 @@ if [[ -d "${docker_dir}" ]]; then fi # VM provisioning end ! +# Copy BTF files. This is a patch for different paths between agent 6 branch code and agent 7 KMT images +if [ -d "/system-probe-tests" ]; then + rsync -avP /system-probe-tests /opt/kmt-ramfs +fi + # Start tests code=0 /test-runner $runner_config || code=$? From a6d0fc41444d6b154c105de8ac96515d150c0ad3 Mon Sep 17 00:00:00 2001 From: Bryce Kahle Date: Thu, 25 Apr 2024 14:44:39 -0700 Subject: [PATCH 06/12] Remove gzip from processed BTFs archive (#25150) * add ninja status for debug * try just tar, no gzip * fix extension * remove outdated comment --- .gitlab/deps_build/deps_build.yml | 4 ++-- .gitlab/package_deps_build/package_deps_build.yml | 4 ++-- tasks/system_probe.py | 7 +++---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.gitlab/deps_build/deps_build.yml b/.gitlab/deps_build/deps_build.yml index 78d5b5941c7e43..0eb5900c391d6f 100644 --- a/.gitlab/deps_build/deps_build.yml +++ b/.gitlab/deps_build/deps_build.yml @@ -90,5 +90,5 @@ build_processed_btfhub_archive: KUBERNETES_CPU_REQUEST: 32 script: - inv -e system-probe.process-btfhub-archive --branch $BTFHUB_ARCHIVE_BRANCH - - $S3_CP_CMD btfs-x86_64.tar.gz $S3_DD_AGENT_OMNIBUS_BTFS_URI/$BTFHUB_ARCHIVE_BRANCH/btfs-x86_64.tar.gz --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers - - $S3_CP_CMD btfs-arm64.tar.gz $S3_DD_AGENT_OMNIBUS_BTFS_URI/$BTFHUB_ARCHIVE_BRANCH/btfs-arm64.tar.gz --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers + - $S3_CP_CMD btfs-x86_64.tar $S3_DD_AGENT_OMNIBUS_BTFS_URI/$BTFHUB_ARCHIVE_BRANCH/btfs-x86_64.tar --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers + - $S3_CP_CMD btfs-arm64.tar $S3_DD_AGENT_OMNIBUS_BTFS_URI/$BTFHUB_ARCHIVE_BRANCH/btfs-arm64.tar --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers diff --git a/.gitlab/package_deps_build/package_deps_build.yml b/.gitlab/package_deps_build/package_deps_build.yml index 7784cfd84c6198..2a62441412ab31 100644 --- a/.gitlab/package_deps_build/package_deps_build.yml +++ b/.gitlab/package_deps_build/package_deps_build.yml @@ -11,8 +11,8 @@ tags: ["arch:amd64"] script: - cd $CI_PROJECT_DIR - - $S3_CP_CMD $S3_DD_AGENT_OMNIBUS_BTFS_URI/$BTFHUB_ARCHIVE_BRANCH/btfs-$ARCH.tar.gz . - - tar -xf btfs-$ARCH.tar.gz + - $S3_CP_CMD $S3_DD_AGENT_OMNIBUS_BTFS_URI/$BTFHUB_ARCHIVE_BRANCH/btfs-$ARCH.tar . + - tar -xf btfs-$ARCH.tar - tar -xf sysprobe-build-outputs.tar.xz - inv -e system-probe.generate-minimized-btfs --source-dir "$CI_PROJECT_DIR/btfs-$ARCH" --output-dir "$CI_PROJECT_DIR/minimized-btfs" --input-bpf-programs "$CI_PROJECT_DIR/pkg/ebpf/bytecode/build/co-re" - cd minimized-btfs diff --git a/tasks/system_probe.py b/tasks/system_probe.py index 45f4a4a4857ba4..b87e6693ef8a94 100644 --- a/tasks/system_probe.py +++ b/tasks/system_probe.py @@ -1621,7 +1621,7 @@ def generate_minimized_btfs( }, ) - ctx.run(f"ninja -f {ninja_file_path}") + ctx.run(f"ninja -f {ninja_file_path}", env={"NINJA_STATUS": "(%r running) (%c/s) (%es) [%f/%t] "}) @task @@ -1678,13 +1678,12 @@ def process_btfhub_archive(ctx, branch="main"): # generate both tarballs for arch in ["x86_64", "arm64"]: btfs_dir = os.path.join(temp_dir, f"btfs-{arch}") - output_path = os.path.join(output_dir, f"btfs-{arch}.tar.gz") + output_path = os.path.join(output_dir, f"btfs-{arch}.tar") # at least one file needs to be moved for directory to exist if os.path.exists(btfs_dir): with ctx.cd(temp_dir): - # gzip ends up being much faster than xz, for roughly the same output file size # include btfs-$ARCH as prefix for all paths - ctx.run(f"tar -czf {output_path} btfs-{arch}") + ctx.run(f"tar -cf {output_path} btfs-{arch}") @task From ef5722e5e026d692a586f6e42a14ce24d7e4818f Mon Sep 17 00:00:00 2001 From: Bryce Kahle Date: Mon, 6 May 2024 10:10:51 -0700 Subject: [PATCH 07/12] add caching of minimized BTFs (#25283) * add caching of minimized BTFs * add --region * fix brackets * use abs path for download * don't need brackets for command --- .gitlab-ci.yml | 1 + .gitlab/binary_build/system_probe.yml | 1 + .gitlab/package_deps_build/package_deps_build.yml | 14 ++++++++++++++ tasks/system_probe.py | 6 ++++++ 4 files changed, 22 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a4047cdd1563f0..fc522a34b6e689 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -139,6 +139,7 @@ variables: S3_CP_OPTIONS: --only-show-errors --region us-east-1 --sse AES256 S3_CP_CMD: aws s3 cp $S3_CP_OPTIONS S3_ARTIFACTS_URI: s3://dd-ci-artefacts-build-stable/$CI_PROJECT_NAME/$CI_PIPELINE_ID + S3_PROJECT_ARTIFACTS_URI: s3://dd-ci-artefacts-build-stable/$CI_PROJECT_NAME S3_PERMANENT_ARTIFACTS_URI: s3://dd-ci-persistent-artefacts-build-stable/$CI_PROJECT_NAME S3_SBOM_STORAGE_URI: s3://sbom-root-us1-ddbuild-io/$CI_PROJECT_NAME/$CI_PIPELINE_ID S3_RELEASE_ARTIFACTS_URI: s3://dd-release-artifacts/$CI_PROJECT_NAME/$CI_PIPELINE_ID diff --git a/.gitlab/binary_build/system_probe.yml b/.gitlab/binary_build/system_probe.yml index dd0e11c7c6d35a..3cbb729d1ec52f 100644 --- a/.gitlab/binary_build/system_probe.yml +++ b/.gitlab/binary_build/system_probe.yml @@ -23,6 +23,7 @@ expire_in: 2 weeks paths: - $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz + - $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz.sum build_system-probe-x64: stage: binary_build diff --git a/.gitlab/package_deps_build/package_deps_build.yml b/.gitlab/package_deps_build/package_deps_build.yml index 2a62441412ab31..c7301662411e84 100644 --- a/.gitlab/package_deps_build/package_deps_build.yml +++ b/.gitlab/package_deps_build/package_deps_build.yml @@ -11,12 +11,26 @@ tags: ["arch:amd64"] script: - cd $CI_PROJECT_DIR + - export BTFS_ETAG=$(aws s3api head-object --region us-east-1 --bucket dd-agent-omnibus --key btfs/$BTFHUB_ARCHIVE_BRANCH/btfs-$ARCH.tar --query ETag --output text | tr -d \") + - export OUTPUTS_HASH=$(sha256sum sysprobe-build-outputs.tar.xz.sum | cut -d' ' -f1) + - export MIN_BTFS_FILENAME=minimized-btfs-$BTFS_ETAG-$OUTPUTS_HASH.tar.xz + - | + # if running all builds, or this is a release branch, skip the cache check + if [[ "$RUN_ALL_BUILDS" != "true" && ! $CI_COMMIT_BRANCH =~ /^[0-9]+\.[0-9]+\.x$/ ]]; then + if aws s3api head-object --region us-east-1 --bucket dd-ci-artefacts-build-stable --key $CI_PROJECT_NAME/btfs/$MIN_BTFS_FILENAME; then + $S3_CP_CMD $S3_PROJECT_ARTIFACTS_URI/btfs/$MIN_BTFS_FILENAME $CI_PROJECT_DIR/minimized-btfs.tar.xz + echo "cached minimized BTFs exist" + exit 0 + fi + fi + # cache does not exist, download processed BTFs and minimize - $S3_CP_CMD $S3_DD_AGENT_OMNIBUS_BTFS_URI/$BTFHUB_ARCHIVE_BRANCH/btfs-$ARCH.tar . - tar -xf btfs-$ARCH.tar - tar -xf sysprobe-build-outputs.tar.xz - inv -e system-probe.generate-minimized-btfs --source-dir "$CI_PROJECT_DIR/btfs-$ARCH" --output-dir "$CI_PROJECT_DIR/minimized-btfs" --input-bpf-programs "$CI_PROJECT_DIR/pkg/ebpf/bytecode/build/co-re" - cd minimized-btfs - tar -cJf $CI_PROJECT_DIR/minimized-btfs.tar.xz * + - $S3_CP_CMD $CI_PROJECT_DIR/minimized-btfs.tar.xz $S3_PROJECT_ARTIFACTS_URI/btfs/$MIN_BTFS_FILENAME variables: KUBERNETES_MEMORY_REQUEST: "6Gi" KUBERNETES_MEMORY_LIMIT: "12Gi" diff --git a/tasks/system_probe.py b/tasks/system_probe.py index b87e6693ef8a94..de294171c4fd92 100644 --- a/tasks/system_probe.py +++ b/tasks/system_probe.py @@ -1871,6 +1871,7 @@ def save_build_outputs(ctx, destfile): absdest = os.path.abspath(destfile) count = 0 + outfiles = [] with tempfile.TemporaryDirectory() as stagedir: with open("compile_commands.json", "r") as compiledb: for outputitem in json.load(compiledb): @@ -1885,8 +1886,13 @@ def save_build_outputs(ctx, destfile): outdir = os.path.join(stagedir, filedir) ctx.run(f"mkdir -p {outdir}") ctx.run(f"cp {outputitem['output']} {outdir}/") + outfiles.append(outputitem['output']) count += 1 if count == 0: raise Exit(message="no build outputs captured") ctx.run(f"tar -C {stagedir} -cJf {absdest} .") + + outfiles.sort() + for outfile in outfiles: + ctx.run(f"sha256sum {outfile} >> {absdest}.sum") From 7444bd61001c9e01be6a40717803ab6a348b0447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20Julia=CC=81n?= Date: Wed, 4 Dec 2024 17:35:04 +0100 Subject: [PATCH 08/12] Use main branch of btfhub --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fc522a34b6e689..5988b37dfeca2a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -151,7 +151,7 @@ variables: INTEGRATION_WHEELS_CACHE_BUCKET: dd-agent-omnibus S3_DD_AGENT_OMNIBUS_LLVM_URI: s3://dd-agent-omnibus/llvm S3_DD_AGENT_OMNIBUS_BTFS_URI: s3://dd-agent-omnibus/btfs - BTFHUB_ARCHIVE_BRANCH: no-kmod + BTFHUB_ARCHIVE_BRANCH: main GENERAL_ARTIFACTS_CACHE_BUCKET_URL: https://dd-agent-omnibus.s3.amazonaws.com S3_DSD6_URI: s3://dsd6-staging RELEASE_VERSION_6: nightly @@ -1131,4 +1131,3 @@ workflow: - .gitlab-ci.yml - .gitlab/**/* compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916 - From 7f99eaaf72401619674d6208e186c7e26620221d Mon Sep 17 00:00:00 2001 From: Bryce Kahle Date: Thu, 24 Oct 2024 12:27:56 -0700 Subject: [PATCH 09/12] increase memory request for minimize BTF jobs (#30433) --- .gitlab/package_deps_build/package_deps_build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab/package_deps_build/package_deps_build.yml b/.gitlab/package_deps_build/package_deps_build.yml index c7301662411e84..869cb16bc4eb51 100644 --- a/.gitlab/package_deps_build/package_deps_build.yml +++ b/.gitlab/package_deps_build/package_deps_build.yml @@ -32,8 +32,9 @@ - tar -cJf $CI_PROJECT_DIR/minimized-btfs.tar.xz * - $S3_CP_CMD $CI_PROJECT_DIR/minimized-btfs.tar.xz $S3_PROJECT_ARTIFACTS_URI/btfs/$MIN_BTFS_FILENAME variables: - KUBERNETES_MEMORY_REQUEST: "6Gi" - KUBERNETES_MEMORY_LIMIT: "12Gi" + KUBERNETES_MEMORY_REQUEST: "64Gi" + KUBERNETES_MEMORY_LIMIT: "64Gi" + KUBERNETES_CPU_REQUEST: 24 artifacts: expire_in: 2 weeks paths: From d74e260b62a605d9181be9f78a6beb430a02a364 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20Julia=CC=81n?= Date: Thu, 5 Dec 2024 10:37:54 +0100 Subject: [PATCH 10/12] Fix path for pre-prepared docker vm disk --- tasks/system_probe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/system_probe.py b/tasks/system_probe.py index de294171c4fd92..4d34741a06b7a4 100644 --- a/tasks/system_probe.py +++ b/tasks/system_probe.py @@ -1768,7 +1768,7 @@ def save_test_dockers(ctx, output_dir, arch, use_crane=False): return # only download images not present in preprepared vm disk - resp = requests.get('https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/docker.ls') + resp = requests.get('https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/docker.ls') docker_ls = {line for line in resp.text.split('\n') if line.strip()} images = _test_docker_image_list() From 91e530e7008ebb3a43b1ee01e5d3a5c990b4feda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20Julia=CC=81n?= Date: Thu, 5 Dec 2024 18:18:16 +0100 Subject: [PATCH 11/12] Fix path for docker image search --- test/new-e2e/system-probe/test/micro-vm-init.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/new-e2e/system-probe/test/micro-vm-init.sh b/test/new-e2e/system-probe/test/micro-vm-init.sh index 97d5769a7ab209..285072df813915 100755 --- a/test/new-e2e/system-probe/test/micro-vm-init.sh +++ b/test/new-e2e/system-probe/test/micro-vm-init.sh @@ -2,15 +2,17 @@ set -eEuxo pipefail runner_config=$@ -docker_dir=/kmt-dockers +docker_dir=/kmt-dockers/src # Add provisioning steps here ! ## Start docker systemctl start docker ## Load docker images if [[ -d "${docker_dir}" ]]; then - find "${docker_dir}" -maxdepth 1 -type f -exec docker load -i {} \; + find "${docker_dir}" -name "*.tar*" -type f -exec /bin/bash -c "echo loading image {} && docker load -i {}" \; fi + +docker images # VM provisioning end ! # Copy BTF files. This is a patch for different paths between agent 6 branch code and agent 7 KMT images From 996216ec4b58d5a4fb3d1d35adfd3d68a7c63b60 Mon Sep 17 00:00:00 2001 From: Usama Saqib Date: Wed, 3 Apr 2024 16:52:20 +0200 Subject: [PATCH 12/12] fix the expected arch suffix for 'kmt-dockers' directory (#24347) --- .gitlab/kernel_matrix_testing/system_probe.yml | 4 ++-- tasks/system_probe.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitlab/kernel_matrix_testing/system_probe.yml b/.gitlab/kernel_matrix_testing/system_probe.yml index 8054a73f90c684..dba6bb80afa0da 100644 --- a/.gitlab/kernel_matrix_testing/system_probe.yml +++ b/.gitlab/kernel_matrix_testing/system_probe.yml @@ -5,7 +5,7 @@ upload_dependencies_sysprobe_x64: needs: ["pull_test_dockers_x64"] rules: !reference [.on_system_probe_or_e2e_changes_or_manual] variables: - ARCH: amd64 + ARCH: x86_64 INSTANCE_TYPE: m5d.metal TEST_COMPONENT: system-probe @@ -43,7 +43,7 @@ upload_dependencies_sysprobe_arm64: pull_test_dockers_x64: extends: .pull_test_dockers variables: - ARCH: amd64 + ARCH: x86_64 pull_test_dockers_arm64: extends: .pull_test_dockers diff --git a/tasks/system_probe.py b/tasks/system_probe.py index 4d34741a06b7a4..850fca0046c92e 100644 --- a/tasks/system_probe.py +++ b/tasks/system_probe.py @@ -1767,6 +1767,10 @@ def save_test_dockers(ctx, output_dir, arch, use_crane=False): if is_windows: return + # crane does not accept 'x86_64' as a valid architecture + if arch == "x86_64": + arch = "amd64" + # only download images not present in preprepared vm disk resp = requests.get('https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/docker.ls') docker_ls = {line for line in resp.text.split('\n') if line.strip()}