Skip to content

Commit

Permalink
Fix incorrect build result with build success + semantic error (#217)
Browse files Browse the repository at this point in the history
The old code incorrectly treats a failure in fixing semantic errors as a
failure in fixing build error, hence it incorrectly labels buildable but
semantically wrong fuzz targets as not buildable.
PoE:
https://llm-exp.oss-fuzz.com/Result-reports/scheduled/2024-04-11-daily-comparison/benchmark/output-fribidi-fribidi_log2vis/index.html

This PR fixes it.
  • Loading branch information
DonggeLiu authored Apr 16, 2024
1 parent d2b1bfa commit b293d64
Showing 1 changed file with 21 additions and 18 deletions.
39 changes: 21 additions & 18 deletions experiment/evaluator.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ class Result:
is_driver_fuzz_err: bool = dataclasses.field(kw_only=True, default=False)
driver_fuzz_err: str = dataclasses.field(kw_only=True, default='')

def __post_init__(self, *args, **kwargs):
def __post_init__(self, *args, **kwargs): # pylint: disable=unused-argument
if self.is_driver_fuzz_err:
self.is_semantic_error = self.is_driver_fuzz_err
if self.driver_fuzz_err:
Expand Down Expand Up @@ -408,11 +408,11 @@ def do_check_target(self, ai_binary: str, target_path: str) -> Result:
while True:
# 1. Evaluating generated driver.
(build_result, run_result, cov_pcs, total_pcs, crashes,
semantic_error) = self._evaluate_generated_fuzz_target(
semantic_check_result) = self._evaluate_generated_fuzz_target(
generated_oss_fuzz_project, target_path, generated_target_name,
llm_fix_count, logger)

gen_succ = build_result.succeeded and not semantic_error.has_err
gen_succ = build_result.succeeded and not semantic_check_result.has_err
if gen_succ:
# Successfully generate the fuzz target.
break
Expand All @@ -428,33 +428,36 @@ def do_check_target(self, ai_binary: str, target_path: str) -> Result:
f'attempt {llm_fix_count}.')
self._fix_generated_fuzz_target(ai_binary, generated_oss_fuzz_project,
target_path, llm_fix_count, build_result,
semantic_error)
semantic_check_result)

# Logs and returns the result.
if gen_succ:
logger.log(f'Successfully built {target_path} with '
f'{self.builder_runner.fixer_model_name} in '
f'{llm_fix_count} iterations.')
else:
logger.log(f'Failed to fix {target_path} with '
if not build_result.succeeded:
logger.log(f'Failed to build {target_path} with '
f'{self.builder_runner.fixer_model_name} in '
f'{llm_fix_count} iterations.')
f'{llm_fix_count} iterations of syntax fixing.')
return logger.return_result(
Result(False, False, 0.0, 0.0, '', '', False, semantic_error.type))
Result(False, False, 0.0, 0.0, '', '', False,
semantic_check_result.type))

logger.log(f'Successfully built {target_path} with '
f'{self.builder_runner.fixer_model_name} in '
f'{llm_fix_count} iterations of syntax fixing.')

if (not run_result or run_result.coverage_summary is None or
run_result.coverage is None):
logger.log(f'Warning: No run_result in {generated_oss_fuzz_project}.')
return logger.return_result(
Result(True, crashes, 0.0, 0.0, '', '', False, semantic_error.type))
Result(True, crashes, 0.0, 0.0, '', '', False,
semantic_check_result.type))

if semantic_error.has_err:
if semantic_check_result.has_err:
logger.log(
f'Warning: {semantic_error.type} in {generated_oss_fuzz_project}.')
f'Warning: Failed to fix sematic error {semantic_check_result.type}'
f' in {generated_oss_fuzz_project}.')
return logger.return_result(
Result(True, crashes, 0.0, 0.0, run_result.coverage_report_path,
run_result.reproducer_path, semantic_error.has_err,
semantic_error.type))
run_result.reproducer_path, semantic_check_result.has_err,
semantic_check_result.type))

# Gets line coverage (diff) details.
coverage_summary = self._load_existing_coverage_summary()
Expand Down Expand Up @@ -482,7 +485,7 @@ def do_check_target(self, ai_binary: str, target_path: str) -> Result:
return logger.return_result(
Result(True, crashes, coverage_percent, coverage_diff,
run_result.coverage_report_path, run_result.reproducer_path,
semantic_error.has_err, semantic_error.type))
semantic_check_result.has_err, semantic_check_result.type))

def _load_existing_coverage_summary(self) -> dict:
"""Load existing summary.json."""
Expand Down

0 comments on commit b293d64

Please sign in to comment.