From eb86298616da150ef7aed28a029e8c91f0420843 Mon Sep 17 00:00:00 2001 From: Giuliano Belinassi Date: Mon, 10 Jun 2024 10:12:39 -0300 Subject: [PATCH] Use debuginfo when extracting information about the packages SUSE packages now strip the .symtab away and store it in the debuginfo, so get the debuginfo for the analysis as well. ulp extract now accepts `-d` to specify a debuginfo. Signed-off-by: Giuliano Belinassi --- scripts/setup_package.sh | 209 ++++++++++++++++++++++++++++++--------- tools/arguments.h | 1 + tools/extract.c | 62 ++++++++++++ tools/ulp.c | 9 +- 4 files changed, 235 insertions(+), 46 deletions(-) diff --git a/scripts/setup_package.sh b/scripts/setup_package.sh index 2ac832ed..72256b4d 100755 --- a/scripts/setup_package.sh +++ b/scripts/setup_package.sh @@ -24,6 +24,7 @@ PROGNAME=`basename "$0"` SLE_VERSION_REGEX="[0-9]{6}" VERSION_REGEX="([0-9\.a-zA-Z]+-$SLE_VERSION_REGEX\.[0-9\.]+[0-9])" PLATFORM= +PRODUCT= URL= PACKAGE= NO_CLEANUP=0 @@ -37,10 +38,29 @@ NO_IPA_CLONES_DOWNLOAD=0 # If this flag is enabled, then download of debuginfo packages will be blocked. NO_DEBUGINFO_DOWNLOAD=0 +# Pushd and popd are not silent. Silence them. +pushd () +{ + command pushd "$@" > /dev/null +} +popd () +{ + command popd "$@" > /dev/null +} + set_url_platform() { PLATFORM=$1 - URL="https://download.suse.de/updates/SUSE/Updates/SLE-Module-Basesystem/$PLATFORM/x86_64/update/x86_64" + PRODUCT=$2 + local element=$3 + + URL="https://download.suse.de/download/ibs/SUSE:/SLE-$PLATFORM:/$PRODUCT/standard" + + if [ "$element" == "src" ]; then + URL="$URL/src" + elif [ "$element" != "ipa-clones" ]; then + URL="$URL/x86_64" + fi } web_get() @@ -89,6 +109,7 @@ get_sle_version_from_package_name() local version=$(echo "$1" | grep -Eo "($SLE_VERSION_REGEX)") + local sle_version=${sle_hash[$version]} if [ "x$sle_version" = "x" ]; then @@ -120,6 +141,7 @@ download_package_list() local url="$URL" local list_path=$1 + echo downloading package list: "$url" web_get "$url" "$list_path" } @@ -211,11 +233,6 @@ get_list_of_debuginfo_packages() local package_name=$(get_name_from_package_name $package) local version=$(get_version_from_package_name $package) - # libopenssl1_1 src comes from openssl. - if [ "$package_name" = "libopenssl1_1" ]; then - package_name="openssl-1_1" - fi - src_package_list="$src_package_list $package_name-debuginfo-$version.x86_64.rpm" done @@ -229,10 +246,8 @@ download_debuginfo_packages() echo $packages - URL="https://download.suse.de/updates/SUSE/Updates/SLE-Module-Basesystem/$PLATFORM/x86_64/update_debug/x86_64/" + set_url_platform $PLATFORM $PRODUCT "debuginfo" parallel_download_packages "$packages" - - URL=$old_url } download_src_packages() @@ -240,10 +255,8 @@ download_src_packages() local packages=$(get_list_of_src_packages "$*") local old_url=$URL - URL="https://download.suse.de/updates/SUSE/Updates/SLE-Module-Basesystem/$PLATFORM/x86_64/update/src/" + set_url_platform $PLATFORM $PRODUCT "src" parallel_download_packages "$packages" - - URL=$old_url } download_ipa_clones() @@ -254,10 +267,8 @@ download_ipa_clones() local sle_ver=$(get_sle_version_from_package_name $1) - URL="https://download.suse.de/download/ibs/SUSE:/SLE-$sle_ver:/Update/standard/" + set_url_platform $PLATFORM $PRODUCT "ipa-clones" parallel_download_packages "$ipa_clones_list" - - URL=$old_url } extract_libs_from_package() @@ -272,22 +283,47 @@ extract_libs_from_package() mkdir -p $PLATFORM/$name/$version cp $package $PLATFORM/$name/$version/$package - cp $src_package $PLATFORM/$name/$version/$src_package - cp $ipa_clones $PLATFORM/$name/$version/$ipa_clones - cp $debuginfo_package $PLATFORM/$name/$version/$debuginfo_package + if [ $? -ne 0 ]; then + echo "error: $package not downloaded." + exit 1 + fi + + if [ $NO_SRC_DOWNLOAD -eq 0 ]; then + cp $src_package $PLATFORM/$name/$version/$src_package + if [ $? -ne 0 ]; then + echo "error: $src_package not downloaded." + exit 1 + fi + fi + + if [ $NO_IPA_CLONES_DOWNLOAD -eq 0 ]; then + cp $ipa_clones $PLATFORM/$name/$version/$ipa_clones + if [ $? -ne 0 ]; then + echo "error: $ipa_clones not downloaded." + exit 1 + fi + fi + + if [ $NO_DEBUGINFO_DOWNLOAD -eq 0 ]; then + cp $debuginfo_package $PLATFORM/$name/$version/$debuginfo_package + if [ $? -ne 0 ]; then + echo "error: $debuginfo not downloaded." + exit 1 + fi + fi cd $PLATFORM/$name/$version mkdir -p binaries cd binaries if [ -f ../$package ]; then - rpm2cpio ../$package | cpio -idmv --quiet + rpm2cpio ../$package | cpio -idm --quiet fi cd .. mkdir -p src cd src if [ -f ../$src_package ]; then - rpm2cpio ../$src_package | cpio -idmv --quiet + rpm2cpio ../$src_package | cpio -idm --quiet tar xf $(ls | grep -E "(\.tar\.xz$|\.tar\.gz$)") fi cd .. @@ -296,7 +332,7 @@ extract_libs_from_package() cd debuginfo if [ -f ../$debuginfo_package ]; then echo "Extracting $debuginfo_package" - rpm2cpio ../$debuginfo_package | cpio -idmv --quiet + rpm2cpio ../$debuginfo_package | cpio -idm --quiet fi cd .. @@ -314,26 +350,98 @@ extract_libs_from_package() cd ../../../ } +# List of .debug files in folder. Stored here for cache reasons. +_LIST_OF_DEBUG="" + +match_so_to_debuginfo() +{ + local so=$1 + local base_so=$(basename $so) + + local list_of_debug=$(echo $_LIST_OF_DEBUG | xargs -n1 | grep -E "$base_so.*\.debug$") + + let num=0 + + # Count how many files we got. + for dbg in $list_of_debug; do + let "num=num+1" + done + + # Assert that we got only one file + if [ $num -ne 1 ]; then + echo "Expected only 1 file matching $base_so, got $num: $list_of_debug" > /dev/stderr + exit 1 + fi + + # Return the file we got. + echo $list_of_debug +} + dump_interesting_info_from_elfs() { - local list_of_sos=$(find $PLATFORM | grep -E "*.so[\.0-9]*$") + pushd $1 + local list_of_sos=$(find . | grep -E ".*\.so[\.0-9]*$") + + # Populate cache of list of .debug + _LIST_OF_DEBUG=$(find . -name "*.debug") - # Extract the relevant so information needed for livepatching. + # Iterate on every so in the folder. for so in $list_of_sos; do - ulp extract $so -o $so.json - done + if [ $NO_DEBUGINFO_DOWNLOAD -eq 0 ]; then + # Get the debuginfo that matches this library. + local debug=$(match_so_to_debuginfo $so) - echo $list_of_sos + if [ "$debug" == "" ]; then + exit 1 + fi + + # Run the ulp extract command on both the library and debuginfo. + echo ulp extract $so -d $debug -o $so.json + ulp extract $so -d $debug -o $so.json + else + # Run the ulp extract command only on the library. + echo ulp extract $so -o $so.json + ulp extract $so -o $so.json + fi + done # Delete all .so we don't need. for so in $list_of_sos; do rm -f $so done + + # Delete all .debug we we don't need. + for debug in $_LIST_OF_DEBUG; do + rm -f $debug + done + + # Delete empty directories left. + find . -type d -empty -delete + + # Invalidate the cache. + _LIST_OF_DEBUG="" + + popd +} + +dump_interesting_info_from_elfs_in_lib() +{ + local platform=$1 + + # Enter in platform folder + pushd $platform + + # Iterate on every version + for dir in $(ls); do + dump_interesting_info_from_elfs $dir + done + popd + } sanitize_platform() { - local platforms="15-SP3 15-SP4" + local platforms="15-SP3 15-SP4 15-SP5" for platform in ${platforms}; do if [ "$PLATFORM" = "$platform" ]; then @@ -440,36 +548,49 @@ parse_program_argv() sanitize_package # Set platform globally - set_url_platform "$PLATFORM" + set_url_platform "$PLATFORM" "GA" } main() { - # Set default URL platform to "15-SP4". - set_url_platform "15-SP4" parse_program_argv $* + # Clean the directory + rm -rf $PLATFORM - download_package_list "/tmp/suse_package_list.html" - local names=$(extract_lib_package_names "/tmp/suse_package_list.html" $PACKAGE) + local all_names="" + local products="GA Update" - parallel_download_packages "$names" + for product in GA Update; do + # Set platform globally + set_url_platform "$PLATFORM" $product - if [ $NO_SRC_DOWNLOAD -eq 0 ]; then - download_src_packages "$names" - fi - if [ $NO_IPA_CLONES_DOWNLOAD -eq 0 ]; then - download_ipa_clones "$names" - fi - if [ $NO_DEBUGINFO_DOWNLOAD -eq 0 ]; then - download_debuginfo_packages "$names" - fi + download_package_list "/tmp/suse_package_list.html" + local names=$(extract_lib_package_names "/tmp/suse_package_list.html" $PACKAGE) + + # Clean the directory + rm -rf $PLATFORM + + parallel_download_packages "$names" + + if [ $NO_SRC_DOWNLOAD -eq 0 ]; then + download_src_packages "$names" + fi + if [ $NO_IPA_CLONES_DOWNLOAD -eq 0 ]; then + download_ipa_clones "$names" + fi + if [ $NO_DEBUGINFO_DOWNLOAD -eq 0 ]; then + download_debuginfo_packages "$names" + fi + + all_names="$all_names $names" + done - for package in $names; do + for package in $all_names; do extract_libs_from_package "$package" done - dump_interesting_info_from_elfs + dump_interesting_info_from_elfs_in_lib $PLATFORM/$PACKAGE # Delete all packages to cleanup. if [ $NO_CLEANUP -ne 1 ]; then diff --git a/tools/arguments.h b/tools/arguments.h index b94ebccc..58cf5f13 100644 --- a/tools/arguments.h +++ b/tools/arguments.h @@ -50,6 +50,7 @@ struct arguments const char *process_wildcard; const char *user_wildcard; const char *prefix; + const char *with_debuginfo; command_t command; int retries; int quiet; diff --git a/tools/extract.c b/tools/extract.c index c97b6950..b33d545f 100644 --- a/tools/extract.c +++ b/tools/extract.c @@ -252,6 +252,10 @@ write_ulp_so_info_json(FILE *stream, const struct ulp_so_info *info) static struct symbol * get_list_of_symbols_in_section(Elf *elf, Elf_Scn *s) { + if (elf == NULL || s == NULL) { + return NULL; + } + struct symbol *symbol_head = NULL; int nsyms, i; @@ -638,12 +642,57 @@ get_symbol_with_name(struct ulp_so_info *info, const char *sym) return symbol; } +struct ulp_so_info *merge_ulp_so_infos(struct ulp_so_info *a, + struct ulp_so_info *b) +{ + /* Sanity check. */ + if (memcmp(a->buildid, b->buildid, sizeof(a->buildid)) != 0) { + char a_buildid[2 * BUILDID_LEN + 1]; + char b_buildid[2 * BUILDID_LEN + 1]; + memcpy(a_buildid, buildid_to_string(a->buildid), sizeof(a_buildid)); + memcpy(b_buildid, buildid_to_string(b->buildid), sizeof(b_buildid)); + + WARN("Attempt to merge symbol lists with distinct buildids\n" + " lib: %s\n" + " debuginfo: %s", a_buildid, b_buildid); + } + + /* Merge both lists. */ + struct symbol **it = &(a->symbols); + for (; *it != NULL; it = &(*it)->next) + ; + + /* Duplicate the list for the sake of avoiding silly memory errors. */ + struct symbol *it2 = b->symbols; + for (; it2 != NULL; it2 = it2->next) { + struct symbol *new = calloc(1, sizeof(struct symbol)); + assert(new && "Error allocating a new symbol element."); + + new->name = strdup(it2->name); + new->offset = it2->offset; + new->size = it2->size; + new->st_info = it2->st_info; + new->st_other = it2->st_other; + new->next = NULL; + + /* Insert into the list. */ + *it = new; + it = &(new->next); + } + + return a; +} + int run_extract(struct arguments *arguments) { /* Path to input livepatch library target. */ const char *input_file = arguments->args[0]; + /* Path to the debuginfo file, which can be used to extract more + information. */ + const char *debuginfo_file = arguments->with_debuginfo; + /* arguments->metadata is what is captured by the -o option. */ const char *output_file = arguments->metadata; @@ -652,6 +701,18 @@ run_extract(struct arguments *arguments) output_file = "out.json"; struct ulp_so_info *info = parse_so_elf(input_file); + struct ulp_so_info *debuginfo = NULL; + + if (info == NULL) { + WARN("error reading elf file %s", input_file); + return 1; + } + + if (debuginfo_file) { + debuginfo = parse_so_elf(debuginfo_file); + info = merge_ulp_so_infos(info, debuginfo); + } + FILE *out; if (!strcmp(output_file, "-")) @@ -666,5 +727,6 @@ run_extract(struct arguments *arguments) fclose(out); release_so_info(info); + release_so_info(debuginfo); return 0; } diff --git a/tools/ulp.c b/tools/ulp.c index 16b7a445..e20a6495 100644 --- a/tools/ulp.c +++ b/tools/ulp.c @@ -171,8 +171,10 @@ static struct argp_option options[] = { "Use this livepatch file\nDefaults to the one described in ARG1", 0 }, { "target", 't', "LIBRARY", 0, "Use this target library\nDefaults to the one described in ARG1", 0 }, - { "color", ULP_OP_COLOR, "yes/no/auto", 0, "Enable/disable colored messages", - 0 }, + { "color", ULP_OP_COLOR, "yes/no/auto", 0, "Enable/disable colored messages", 0 }, + { 0, 0, 0, 0, "extract command only:", 0 }, + { "with-debuginfo", 'd', "DEBUGINFO", 0, + "Use debuginfo information for symbolextraction", 0 }, { 0 } }; @@ -344,6 +346,9 @@ parser(int key, char *arg, struct argp_state *state) case 'R': arguments->prefix = arg; break; + case 'd': + arguments->with_debuginfo = arg; + break; case ULP_OP_REVERT_ALL: arguments->library = get_basename(arg); break;