Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

experimental: refactor to simplify and compartmentalise #454

Merged
merged 14 commits into from
Jul 9, 2024
Merged

experimental: refactor to simplify and compartmentalise #454

Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
8ec1334
experimental: use absolute path for oss-fuzz
DavidKorczynski Jul 8, 2024
f250edc
experimental: fix logging bug
DavidKorczynski Jul 8, 2024
f7f65a9
experimental: refactor runner
DavidKorczynski Jul 8, 2024
50c0378
experimental: refactor manager
DavidKorczynski Jul 8, 2024
a96f299
remove unused code
DavidKorczynski Jul 8, 2024
caec466
fix logger bugs
DavidKorczynski Jul 8, 2024
49720e5
remove unused code
DavidKorczynski Jul 8, 2024
0dab43d
add wrapper class for auto builders
DavidKorczynski Jul 8, 2024
e411bca
fix style
DavidKorczynski Jul 8, 2024
4b43d81
fix types
DavidKorczynski Jul 8, 2024
3e8fbed
experimental: facelift string formatting
DavidKorczynski Jul 8, 2024
75dd414
fix styling
DavidKorczynski Jul 8, 2024
c4db325
nit
DavidKorczynski Jul 8, 2024
5232819
address review
DavidKorczynski Jul 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 44 additions & 38 deletions experimental/c-cpp/build_generator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
import shutil
import subprocess
from abc import abstractmethod
from typing import Any, Dict, Iterator, List, Tuple
from typing import Dict, Iterator, List, Tuple

import manager

Expand All @@ -36,6 +36,17 @@ def __init__(self):
self.heuristic_id = ''


class BuildWorker:
"""Keeper of data on auto generated builds."""

def __init__(self, build_suggestion: AutoBuildContainer):
self.build_suggestion: AutoBuildContainer = build_suggestion
self.build_script: str = ''
self.build_directory: str = ''
self.executable_files_build: Dict[str, List[str]] = {}
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: dict[str, list[str]]

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can't -- this code runs in the oss-fuzz base images which relies on Python 3.8 and the typing is a Python3.9+ thing (see initial PR comment)

self.base_fuzz_build: bool = False


class AutoBuildBase:
"""Base class for auto builders."""

Expand Down Expand Up @@ -454,9 +465,9 @@ def extract_defensive_options(self) -> str:
if option['type'] != 'BOOL':
continue
if 'STATIC' in option['name'] and option['default'] != 'ON':
cmake_string += '-D%s=ON ' % (option['name'])
cmake_string += f'-D{option["name"]}=ON '
elif option['default'] != 'OFF':
cmake_string += '-D%s=OFF ' % (option['name'])
cmake_string += f'-D{option["name"]}=OFF '
return cmake_string


Expand Down Expand Up @@ -516,7 +527,7 @@ def steps_to_build(self):
opt1 = [
'mkdir fuzz-build',
'cd fuzz-build',
'cmake %s ../' % (' '.join(cmake_opts)),
f'cmake {" ".join(cmake_opts)} ../',
'make V=1 || true',
]
build_container2 = AutoBuildContainer()
Expand All @@ -528,7 +539,7 @@ def steps_to_build(self):
opt_static = [
'mkdir fuzz-build',
'cd fuzz-build',
'cmake %s ../' % (' '.join(cmake_opts)),
f'cmake {" ".join(cmake_opts)} ../',
'sed -i \'s/SHARED/STATIC/g\' ../CMakeLists.txt',
'make V=1 || true',
]
Expand All @@ -541,13 +552,13 @@ def steps_to_build(self):
option_values = []
for option in self.cmake_options:
if 'BUILD_SHARED_LIBS' == option:
option_values.append('-D%s=OFF' % (option))
option_values.append(f'-D{option}=OFF')
elif 'BUILD_STATIC' == option:
option_values.append('-D%s=ON' % (option))
option_values.append(f'-D{option}=ON')
elif 'BUILD_SHARED' == option:
option_values.append('-D%s=OFF' % (option))
option_values.append(f'-D{option}=OFF')
elif 'ENABLE_STATIC' == option:
option_values.append('-D%s=ON' % (option))
option_values.append(f'-D{option}=ON')

if len(option_values) > 0:
option_string = ' '.join(option_values)
Expand All @@ -558,7 +569,7 @@ def steps_to_build(self):
bopt = [
'mkdir fuzz-build',
'cd fuzz-build',
'cmake %s %s ../' % (' '.join(cmake_default_options), option_string),
f'cmake {" ".join(cmake_default_options)} {option_string} ../',
'make V=1',
]
build_container3 = AutoBuildContainer()
Expand All @@ -571,15 +582,15 @@ def steps_to_build(self):
option_values = []
for option in self.cmake_options:
if 'BUILD_SHARED_LIBS' == option:
option_values.append('-D%s=OFF' % (option))
option_values.append(f'-D{option}=OFF')
elif 'BUILD_STATIC' == option:
option_values.append('-D%s=ON' % (option))
option_values.append(f'-D{option}=ON')
elif 'BUILD_SHARED' == option:
option_values.append('-D%s=OFF' % (option))
option_values.append(f'-D{option}=OFF')
elif 'ENABLE_STATIC' == option:
option_values.append('-D%s=ON' % (option))
option_values.append(f'-D{option}=ON')
elif 'BUILD_TESTS' in option:
option_values.append('-D%s=ON' % (option))
option_values.append(f'-D{option}=ON')

if len(option_values) > 0:
option_string = ' '.join(option_values)
Expand All @@ -590,7 +601,7 @@ def steps_to_build(self):
bopt = [
'mkdir fuzz-build',
'cd fuzz-build',
'cmake %s %s ../' % (' '.join(cmake_default_options), option_string),
f'cmake {" ".join(cmake_default_options)} {option_string} ../',
'make V=1',
]
build_container4 = AutoBuildContainer()
Expand Down Expand Up @@ -666,9 +677,9 @@ def get_all_binary_files_from_folder(path: str) -> Dict[str, List[str]]:
def wrap_build_script(test_dir: str, build_container: AutoBuildContainer,
abspath_of_target: str) -> str:
build_script = '#!/bin/bash\n'
build_script += 'rm -rf /%s\n' % (test_dir)
build_script += 'cp -rf %s %s\n' % (abspath_of_target, test_dir)
build_script += 'cd %s\n' % (test_dir)
build_script += f'rm -rf /{test_dir}\n'
build_script += f'cp -rf {abspath_of_target} {test_dir}\n'
build_script += f'cd {test_dir}\n'
for cmd in build_container.list_of_commands:
build_script += cmd + '\n'

Expand Down Expand Up @@ -707,8 +718,8 @@ def extract_build_suggestions(


def raw_build_evaluation(
all_build_scripts: List[Tuple[str, str, AutoBuildContainer]],
initial_executable_files: Dict[str, List[str]]) -> Dict[str, Any]:
all_build_scripts: List[Tuple[str, str, AutoBuildContainer]]
) -> Dict[str, BuildWorker]:
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: list[tuple[str, str, AutoBuildContainer]], dict[str, BuildWorker]

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can't -- this code runs in the oss-fuzz base images which relies on Python 3.8 and the typing is a Python3.9+ thing (see initial PR comment)

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oh I see. thanks!

"""Run each of the build scripts and extract any artifacts build by them."""
build_results = {}
for build_script, test_dir, build_suggestion in all_build_scripts:
Expand All @@ -727,22 +738,17 @@ def raw_build_evaluation(
# to running the build.
binary_files_build = get_all_binary_files_from_folder(test_dir)

new_binary_files = {
'static-libs': [],
'dynamic-libs': [],
'object-files': []
}
for key, bfiles in binary_files_build.items():
for bfile in bfiles:
if bfile not in initial_executable_files[key]:
new_binary_files[key].append(bfile)

logger.info('Static libs found %s', str(new_binary_files))

# binary_files_build['static-libs'])
build_results[test_dir] = {
'build-script': build_script,
'executables-build': binary_files_build,
'auto-build-setup': (build_script, test_dir, build_suggestion)
}
#build_results[test_dir] = {
# 'build-script': build_script,
# 'executables-build': binary_files_build,
# 'auto-build-setup': (build_script, test_dir, build_suggestion)
#}
DavidKorczynski marked this conversation as resolved.
Show resolved Hide resolved
build_worker = BuildWorker(build_suggestion)
build_worker.build_script = build_script
build_worker.executable_files_build = binary_files_build
build_worker.build_directory = test_dir
DavidKorczynski marked this conversation as resolved.
Show resolved Hide resolved

build_results[test_dir] = build_worker

return build_results
Loading