From bb27eca2a6e9cdfef99ecf6073455b5d6b5fb34a Mon Sep 17 00:00:00 2001 From: Travis Dart Date: Sat, 11 Jul 2020 01:04:01 -0500 Subject: [PATCH] Test coverage improvements. Updated README.md --- README.md | 71 ++++++++++++++++-- formstorm/FormElement.py | 22 +++--- formstorm/FormTest.py | 14 ++-- tests/.coveragerc | 2 +- tests/fstest/settings.py | 2 +- tests/fstestapp/test.py | 73 ++++++++++++++++--- tests/misctestsapp/__init__.py | 0 tests/misctestsapp/forms.py | 13 ++++ tests/misctestsapp/migrations/0001_initial.py | 21 ++++++ tests/misctestsapp/migrations/__init__.py | 0 tests/misctestsapp/models.py | 5 ++ tests/misctestsapp/tests.py | 63 ++++++++++++++++ 12 files changed, 249 insertions(+), 37 deletions(-) create mode 100644 tests/misctestsapp/__init__.py create mode 100644 tests/misctestsapp/forms.py create mode 100644 tests/misctestsapp/migrations/0001_initial.py create mode 100644 tests/misctestsapp/migrations/__init__.py create mode 100644 tests/misctestsapp/models.py create mode 100644 tests/misctestsapp/tests.py diff --git a/README.md b/README.md index 14752b2..0a7421e 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # FormStorm (v1.0.0) -A library to test Django forms by trying (almost) every combination of valid and invalid input. Rather than testing each field's validation independently, formstorm tests all of the fields' validation simultaneously to ensure that there is no interdependence between fields. +A library to test Django forms by trying (almost) every combination of valid and invalid input. Rather than testing each field's validation independently, formstorm tests all of the fields' validation simultaneously to ensure the form doesn't have unintended interdependence between fields. -Note that Version 1 only verifies that there is no interdependence between fields. The next version will be able to test forms that have multi-field and interdependent validation. +Currently, FormStorm supports testing pre-defined good/bad values, multi-field validation, and single-field uniqueness constraints. ## Example: @@ -82,7 +82,7 @@ An example showing how to use different field types can be found in [tests/fstes Basically, all fields work as above, with the exception of ForeignKey and Many2Many fields whose values must be specified with `Q()` objects. Also, example values for multi-valued fields (such as Many2Many) can be created with the `every_combo()` function which returns every combination of the Many2Many options. -To test the validation of multiple fields, +Validating multi-field constraints can be tested by specifying the values (as a dictionary) along with the expected results. For example, if the "title" and "subtitle" fields can't have a combined length greater than 150 characters, we can test this constraint as below: additional_values = [ ({'title': "A"*100, 'subtitle': "A"*50}, True), @@ -97,7 +97,68 @@ To test the validation of multiple fields, ## TODO: -- Test to ensure that uniqueness constraints work. - Some provision for this feature has already been made, but it hasn't been fully implemented yet. -- End-to-end testing (with Selenium): This is partially implemented, and all of the necessary FormStorm functions have been abstracted. Just need to subclass FormTest and fully implement. + +- End-to-end testing (with Selenium): This is partially implemented, and most of the necessary FormStorm functions have been abstracted. Just need to subclass FormTest and fully implement. - Tests for DRF Serializers. "SerializerStorm" - Set up CI +- Handle the obscure, "long tail" cases by implementing a framework to define tests for any type of constraint (such as multi-column uniqueness constraints). To do this, a "sub-test" would define one or more form submissions and the (boolean) result expected. Sub-tests would be combined with each other and with the standard combinatorial mix of good/bad values so that all fields are tested simultaneously. A tentative definition of the sub-tests is below: + + + sub_tests = [ + { # Sub-test 1 + field_names=["field1","field2",..."fieldN"], + submissions = [ Each sub-test consists of multilpe submissions. + ( # Submission 1 + value1, # the value for field1 + value2, # the value for field2 + ... + valueN, # the value for fieldN + result, # the expected result of the submission + ), + (...), # Submission 2 + ... + (...) # Submission N + ] + }, + {...}, # Sub-test 2 + ... + {...} # Sub-test N + ] + +For example, suppose a model has two fields that have a multi-column uniqueness constraint: + + class SomeModel(models.Model): + field1 = models.TextField() + field2 = models.TextField() + field3 = models.TextField() + ... + fieldN = models.TextField() + + class Meta: + constraints = [ + UniqueConstraint( + fields=['field1', 'field2'], name='unique_together' + ) + ] + +The sub-test to test this constraint would be defined like this: + + class SomeModelFormTest(FormTest): + form = SomeModelForm + sub_tests = [ + { + field_names=["field1","field2"], + submissions = [ + ("a","a", True), + ("a","b", True), # Duplicate values in one column are fine + ("b","a", True), # ... as are duplicates in the other column + ("a","a", False) # ... but the same values again should fail + ] + } + ] + field3 = FormElement(good=[...], bad=[...]) + ... + fieldN = FormElement(good=[...], bad=[...]) + +The advantage of this is that we can define a test only for the fields affected by a constraint, and have values for the other fields supplied by the normal good/bad value tests. + diff --git a/formstorm/FormElement.py b/formstorm/FormElement.py index d303ec8..a922a0c 100644 --- a/formstorm/FormElement.py +++ b/formstorm/FormElement.py @@ -1,13 +1,13 @@ from itertools import chain from django.db.models import Q -from six import text_type +# from six import text_type class FormElement(object): def __init__(self, good=[], bad=[], is_unique=False): self.bad = bad self.good = good - self.is_unique = is_unique + self.is_unique = is_unique def build_iterator(self, form, field_name, is_e2e): """ @@ -26,19 +26,21 @@ def build_iterator(self, form, field_name, is_e2e): def _get_pk_for_q(q_object): # 1st: Find the model that the field points to. form_field = form._meta.model._meta.get_field(field_name) - try: # Python 3 - ref_model = form_field.remote_field.model - except AttributeError: - ref_model = form_field.rel.to + ref_model = form_field.remote_field.model # 2nd: Get the object that the Q object points to. ref_object = ref_model.objects.get(q_object) # 3rd: Get the identifier for that object. - if is_e2e: # If we're doing an e2e test, reference by name. - return text_type(ref_object) - else: - return ref_object.pk + + # If we're doing an e2e test, reference by name. + # (e2e is not implemented yet) + # if is_e2e: + # return text_type(ref_object) + # else: + # return ref_object.pk + + return ref_object.pk def _replace_all_q(value_list): for i, g in enumerate(value_list): diff --git a/formstorm/FormTest.py b/formstorm/FormTest.py index 59dbd69..236c3e4 100644 --- a/formstorm/FormTest.py +++ b/formstorm/FormTest.py @@ -24,7 +24,7 @@ def _build_elements(self, fields_to_ignore=[]): continue # Filter out this class's FormElement properties - form_element = getattr(self, e) + form_element = getattr(self, e) if type(form_element) is FormElement: elements[e] = form_element.build_iterator( is_e2e=self.is_e2e, @@ -123,28 +123,24 @@ def run(self): if self._is_modelform: if not has_run_uniqueness_test and form_is_good: - # There's no way to verify that the uniqueness constraint + # There's no way to verify that the uniqueness constraint # was the one that triggered the error. However, if the # form was previously valid, and now it's not, and we've # submitted the same input, then we can conclude that # non-uniqueness was the issue. self.submit_form(form_values) assert not self.is_good() + # Normally, we only test for valid/invalid, but let's + # do a little extra and verify which fields caused errors. for field in self.unique_elements: assert self.bound_form.has_error(field) has_run_uniqueness_test = True transaction.savepoint_rollback(sid) - + if should_run_uniqueness_test and not has_run_uniqueness_test: # If we make it to the end without having run the uniqueness test, # then it must be because no good input was specified. raise RuntimeError( "Good input must be given to run uniqueness test." ) - - def run_uniqueness_tests(self): - pass - - def run_individual_tests(self): - self._run(is_uniqueness_test=False) diff --git a/tests/.coveragerc b/tests/.coveragerc index 644a90f..4a9abd4 100644 --- a/tests/.coveragerc +++ b/tests/.coveragerc @@ -1,7 +1,7 @@ [run] branch = true omit = fstest/wsgi.py, manage.py -source = . +source = .,../formstorm [report] show_missing = true diff --git a/tests/fstest/settings.py b/tests/fstest/settings.py index 331f490..74ecfd5 100644 --- a/tests/fstest/settings.py +++ b/tests/fstest/settings.py @@ -36,7 +36,7 @@ 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', 'fstestapp', - 'minimalapp' + 'minimalapp', 'misctestsapp' ] MIDDLEWARE = [ diff --git a/tests/fstestapp/test.py b/tests/fstestapp/test.py index 32fef9c..54fecbb 100644 --- a/tests/fstestapp/test.py +++ b/tests/fstestapp/test.py @@ -1,3 +1,4 @@ +import sys from formstorm import FormTest, FormElement from formstorm.iterhelpers import every_combo, dict_combo from .forms import BookForm @@ -6,42 +7,78 @@ from django.db.models import Q +# class BookFormTest(FormTest): +# form = BookForm +# title = FormElement( +# good=["Moby Dick"], +# bad=[None, "", "A"*101], +# is_unique=True +# ) +# subtitle = FormElement( +# good=[None, "", "or The Whale"], +# bad=["A"*101] +# ) +# author = FormElement( +# good=[Q(name="Herman Melville")], +# bad=[None, "", -1] +# ) +# is_fiction = FormElement( +# good=[True, False, None, "", -1, "A"], +# bad=[] # Boolean input is either truthy or falsy, so nothing is bad. +# ) +# pages = FormElement( +# good=[0, 10, 100], +# bad=[None, "", "A"] +# ) +# genre = FormElement( +# good=every_combo([ +# Q(name="Mystery"), +# Q(name="History"), +# Q(name="Humor") +# ]), +# bad=[None] +# ) +# additional_values = [ +# ({'title': "A"*100, 'subtitle': "A"*50}, True), +# ({'title': "A"*50, 'subtitle': "A"*100}, True), +# ({'title': "A"*100, 'subtitle': "A"*51}, False), +# ({'title': "A"*51, 'subtitle': "A"*100}, False), +# ] + + class BookFormTest(FormTest): form = BookForm title = FormElement( good=["Moby Dick"], - bad=[None, "", "A"*101], + bad=[None], is_unique=True ) subtitle = FormElement( - good=[None, "", "or The Whale"], + good=[None], bad=["A"*101] ) author = FormElement( good=[Q(name="Herman Melville")], - bad=[None, "", -1] + bad=[None] ) is_fiction = FormElement( - good=[True, False, None, "", -1, "A"], + good=[True], bad=[] # Boolean input is either truthy or falsy, so nothing is bad. ) pages = FormElement( - good=[0, 10, 100], - bad=[None, "", "A"] + good=[100], + bad=[None] ) genre = FormElement( good=every_combo([ Q(name="Mystery"), - Q(name="History"), Q(name="Humor") ]), bad=[None] ) additional_values = [ ({'title': "A"*100, 'subtitle': "A"*50}, True), - ({'title': "A"*50, 'subtitle': "A"*100}, True), ({'title': "A"*100, 'subtitle': "A"*51}, False), - ({'title': "A"*51, 'subtitle': "A"*100}, False), ] @@ -78,7 +115,14 @@ def test_dict_combo(self): {'a': 'C', 'b': 1, 'c': None}, ] - assert list(x) == y + x = list(x) + + # Compensate for non-deterministic dict insertion if using python <3.7. + if sys.version_info[:2] < (3, 7): + x.sort() + y.sort() + + assert x == y def test_dict_combo_with_base_dict(self): x = dict_combo({ @@ -108,7 +152,14 @@ def test_dict_combo_with_base_dict(self): {'a': 'C', 'b': 1, 'c': None, "d": "A", "e": 0, 'f': True}, ] - assert list(x) == y + x = list(x) + + # See note above. + if sys.version_info[:2] < (3, 7): + x.sort() + y.sort() + + assert x == y class BookTestCase(TestCase): diff --git a/tests/misctestsapp/__init__.py b/tests/misctestsapp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/misctestsapp/forms.py b/tests/misctestsapp/forms.py new file mode 100644 index 0000000..81b3a53 --- /dev/null +++ b/tests/misctestsapp/forms.py @@ -0,0 +1,13 @@ +from django.forms import ModelForm +from minimalapp.models import Book +from django import forms + + +class NameForm(forms.Form): + your_name = forms.CharField(label='Your name', max_length=100) + + +class BookForm(ModelForm): + class Meta: + model = Book + exclude = [] diff --git a/tests/misctestsapp/migrations/0001_initial.py b/tests/misctestsapp/migrations/0001_initial.py new file mode 100644 index 0000000..2f714d1 --- /dev/null +++ b/tests/misctestsapp/migrations/0001_initial.py @@ -0,0 +1,21 @@ +# Generated by Django 3.0.5 on 2020-07-09 04:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Book', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(max_length=100)), + ], + ), + ] diff --git a/tests/misctestsapp/migrations/__init__.py b/tests/misctestsapp/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/misctestsapp/models.py b/tests/misctestsapp/models.py new file mode 100644 index 0000000..2b9fd68 --- /dev/null +++ b/tests/misctestsapp/models.py @@ -0,0 +1,5 @@ +from django.db import models + + +class Book(models.Model): + title = models.CharField(max_length=100, blank=False, null=False) diff --git a/tests/misctestsapp/tests.py b/tests/misctestsapp/tests.py new file mode 100644 index 0000000..ccdc103 --- /dev/null +++ b/tests/misctestsapp/tests.py @@ -0,0 +1,63 @@ +from django.test import TestCase +from formstorm import FormTest, FormElement +from .forms import NameForm, BookForm + + +class NameFormTest(FormTest): + form = NameForm + your_name = FormElement( + good=["or The Whale", "", None], + bad=["A"*101], + is_unique=True + ) + + +class BookFormTest(FormTest): + form = BookForm + title = FormElement( + bad=["A"*101], + is_unique=True + ) + + +class BadFormTest(FormTest): + """ + Make sure an error is thrown when we expect a bad value to be valid. + """ + form = BookForm + title = FormElement( + good=["A"*101], + ) + + +class NameTestCase(TestCase): + def setUp(self): + self.theNameFormTest = NameFormTest() + + def test_name_form(self): + with self.assertRaises(RuntimeError) as context: + self.theNameFormTest.run() + + expected_message = 'Uniqueness tests can only be run on ModelForms' + assert str(context.exception) == expected_message + + +class BookTestCase(TestCase): + def setUp(self): + self.theBookFormTest = BookFormTest() + + def test_book_form(self): + with self.assertRaises(RuntimeError) as context: + self.theBookFormTest.run() + + expected_message = 'Good input must be given to run uniqueness test.' + assert str(context.exception) == expected_message + + +class BadTestCase(TestCase): + def setUp(self): + self.theBadFormTest = BadFormTest() + + def test_book_form(self): + with self.assertRaises(AssertionError): + self.theBadFormTest.run()