Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow checks with no inputs for property-based testing #67

Merged
merged 1 commit into from
Nov 28, 2024
Merged

Allow checks with no inputs for property-based testing #67

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 51 additions & 14 deletions src/scwidgets/check/_check.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,13 +67,16 @@ class Check:
Callable[[FunOutParamsT, FunOutParamsT], str],
Callable[[FingerprintT, FingerprintT], str],
Callable[[FunOutParamsT], str],
Callable[[], str],
]

def __init__(
self,
function_to_check: Callable[..., FunOutParamsT],
asserts: Union[List[AssertFunT], AssertFunT],
inputs_parameters: Union[List[Dict[str, FunInParamT]], Dict[str, FunInParamT]],
inputs_parameters: Optional[
Union[List[Dict[str, FunInParamT]], Dict[str, FunInParamT]]
] = None,
outputs_references: Optional[Union[List[tuple], tuple]] = None,
fingerprint: Optional[
Callable[[Check.FunOutParamsT], Check.FingerprintT]
Expand All @@ -83,29 +86,42 @@ def __init__(
):
self._function_to_check = function_to_check
self._asserts = []
self._nullvariate_asserts: List[Callable[[], str]] = []
self._univariate_asserts: List[Callable[[tuple], str]] = []
self._bivariate_asserts = []
if not (isinstance(asserts, list)):
asserts = [asserts]

for i, assert_f in enumerate(asserts):
n_positional_arguments = len(
nb_positional_arguments = len(
[
parameters
for parameters in inspect.signature(assert_f).parameters.values()
if parameters.default is inspect._empty
]
)
self._asserts.append(assert_f)
if n_positional_arguments == 1:
if nb_positional_arguments == 0:
self._nullvariate_asserts.append(assert_f) # type: ignore[arg-type]
elif nb_positional_arguments == 1:
if inputs_parameters is None:
raise ValueError(
"For functions taking two input arguments we need "
"inputs_parameters."
)
# type checker cannot infer type change
self._univariate_asserts.append(assert_f) # type: ignore[arg-type]
elif n_positional_arguments == 2:
elif nb_positional_arguments == 2:
if inputs_parameters is None or outputs_references is None:
raise ValueError(
"For functions taking two input arguments we need "
"inputs_parameters and outputs_references."
)
self._bivariate_asserts.append(assert_f)
else:
raise ValueError(
f"Only assert function with 1 or 2 positional arguments are allowed"
f"but assert function {i} has {n_positional_arguments} positional"
f"but assert function {i} has {nb_positional_arguments} positional"
f"arguments"
)

Expand All @@ -114,7 +130,7 @@ def __init__(
if isinstance(inputs_parameters, dict):
inputs_parameters = [inputs_parameters]

if outputs_references is not None:
if inputs_parameters is not None and outputs_references is not None:
if isinstance(outputs_references, tuple):
outputs_references = [outputs_references]
assert len(inputs_parameters) == len(outputs_references), (
Expand All @@ -123,8 +139,10 @@ def __init__(
f"[{len(inputs_parameters)} != {len(outputs_references)}]."
)

self._inputs_parameters = inputs_parameters
self._outputs_references = outputs_references
self._inputs_parameters = [] if inputs_parameters is None else inputs_parameters
self._outputs_references = (
[] if outputs_references is None else outputs_references
)
self._fingerprint = fingerprint
self._suppress_fingerprint_asserts = suppress_fingerprint_asserts
self._stop_on_assert_error_raised = stop_on_assert_error_raised
Expand All @@ -145,6 +163,10 @@ def fingerprint(self):
def asserts(self):
return deepcopy(self._asserts)

@property
def nullvariate_asserts(self):
return deepcopy(self._nullvariate_asserts)

@property
def univariate_asserts(self):
return deepcopy(self._univariate_asserts)
Expand All @@ -163,7 +185,9 @@ def outputs_references(self):

@property
def nb_conducted_asserts(self):
return len(self._asserts) * len(self._inputs_parameters)
return len(self._asserts) * len(self._inputs_parameters) + len(
self._nullvariate_asserts
)

def compute_outputs(self):
outputs = []
Expand Down Expand Up @@ -200,20 +224,31 @@ def check_function(self) -> CheckResult:
)

check_result = CheckResult()

for assert_f in self._nullvariate_asserts:
try:
assert_result = assert_f()
check_result.append(assert_result, assert_f, {})
except Exception:
excution_info = sys.exc_info()
check_result.append(excution_info, assert_f, {})
if self._stop_on_assert_error_raised:
return check_result

for i, input_parameters in enumerate(self._inputs_parameters):
output = self._function_to_check(**input_parameters)
if not (isinstance(output, tuple)):
output = (output,)

for assert_f in self._univariate_asserts:
for uni_assert_f in self._univariate_asserts:
try:
assert_result = assert_f(output)
assert_result = uni_assert_f(output)
check_result.append(assert_result, uni_assert_f, input_parameters)
except Exception:
excution_info = sys.exc_info()
check_result.append(excution_info, assert_f, input_parameters)
check_result.append(excution_info, uni_assert_f, input_parameters)
if self._stop_on_assert_error_raised:
return check_result
check_result.append(assert_result, assert_f, input_parameters)

if self._fingerprint is not None:
try:
Expand All @@ -227,7 +262,9 @@ def check_function(self) -> CheckResult:
" most likely because your output type is wrong."
)
excution_info = sys.exc_info()
check_result.append(excution_info, assert_f, input_parameters)
check_result.append(
excution_info, self._fingerprint, input_parameters
)
return check_result

if not (isinstance(output, tuple)):
Expand Down
4 changes: 2 additions & 2 deletions src/scwidgets/check/_widget_check_registry.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ def _add_check_from_check(self, checks: Union[List[Check], Check]):
def _add_check_from_check_parameters(
self,
asserts: Union[List[Check.AssertFunT], Check.AssertFunT],
inputs_parameters: Union[List[dict], dict],
inputs_parameters: Optional[Union[List[dict], dict]] = None,
outputs_references: Optional[
Union[List[Check.FunOutParamsT], Check.FunOutParamsT]
] = None,
Expand Down Expand Up @@ -192,7 +192,7 @@ def add_check(
self,
widget: CheckableWidget,
asserts: Union[List[Check.AssertFunT], Check.AssertFunT],
inputs_parameters: Union[List[dict], dict],
inputs_parameters: Optional[Union[List[dict], dict]] = None,
outputs_references: Optional[
Union[List[Check.FunOutParamsT], Check.FunOutParamsT]
] = None,
Expand Down
22 changes: 22 additions & 0 deletions tests/test_check.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,24 @@ def test_assert_invalid_output_parameter_dtype():
) in result.message()


def no_param_check(failing):

def function_to_check(parameter):
print("SomeText")
return parameter * 2

def custom_assert() -> str:
# property-based tests with the student code
if failing:
return "Check does not fulfill property"
return ""

return Check(
function_to_check=function_to_check,
asserts=[custom_assert],
)


def single_param_check(use_fingerprint=False, failing=False, buggy=False):
if buggy:

Expand Down Expand Up @@ -175,6 +193,7 @@ class TestCheck:
@pytest.mark.parametrize(
"check",
[
no_param_check(failing=False),
single_param_check(use_fingerprint=False, failing=False),
multi_param_check(use_fingerprint=False, failing=False),
single_param_check(use_fingerprint=True, failing=False),
Expand Down Expand Up @@ -205,6 +224,7 @@ def test_compute_and_set_references(self, check):
@pytest.mark.parametrize(
"check",
[
no_param_check(failing=True),
single_param_check(use_fingerprint=False, failing=True),
multi_param_check(use_fingerprint=False, failing=True),
single_param_check(use_fingerprint=True, failing=True),
Expand Down Expand Up @@ -281,6 +301,7 @@ class TestCheckRegistry:
@pytest.mark.parametrize(
"checks",
[
[no_param_check(failing=False)],
[
single_param_check(use_fingerprint=False, failing=False),
single_param_check(use_fingerprint=True, failing=False),
Expand Down Expand Up @@ -356,6 +377,7 @@ def test_compute_and_set_all_references(self, checks):
@pytest.mark.parametrize(
"checks",
[
[no_param_check(failing=True)],
[
single_param_check(use_fingerprint=False, failing=True),
single_param_check(use_fingerprint=True, failing=True),
Expand Down
Loading