Test coverage improvements. Updated README.md

README.md

@@ -1,8 +1,8 @@
 # FormStorm (v1.0.0)
 
-A library to test Django forms by trying (almost) every combination of valid and invalid input. Rather than testing each field's validation independently, formstorm tests all of the fields' validation simultaneously to ensure that there is no interdependence between fields.
+A library to test Django forms by trying (almost) every combination of valid and invalid input. Rather than testing each field's validation independently, formstorm tests all of the fields' validation simultaneously to ensure the form doesn't have unintended interdependence between fields.
 
-Note that Version 1 only verifies that there is no interdependence between fields. The next version will be able to test forms that have multi-field and interdependent validation.
+Currently, FormStorm supports testing pre-defined good/bad values, multi-field validation, and single-field uniqueness constraints.
 
 ## Example:
 
@@ -82,7 +82,7 @@ An example showing how to use different field types can be found in [tests/fstes
 
 Basically, all fields work as above, with the exception of ForeignKey and Many2Many fields whose values must be specified with `Q()` objects. Also, example values for multi-valued fields (such as Many2Many) can be created with the `every_combo()` function which returns every combination of the Many2Many options.
 
-To test the validation of multiple fields, 
+Validating multi-field constraints can be tested by specifying the values (as a dictionary) along with the expected results. For example, if the "title" and "subtitle" fields can't have a combined length greater than 150 characters, we can test this constraint as below:
 
     additional_values = [
         ({'title': "A"*100, 'subtitle': "A"*50}, True),
@@ -97,7 +97,68 @@ To test the validation of multiple fields,
 
 ## TODO:
 
-- Test to ensure that uniqueness constraints work. - Some provision for this feature has already been made, but it hasn't been fully implemented yet. 
-- End-to-end testing (with Selenium): This is partially implemented, and all of the necessary FormStorm functions have been abstracted. Just need to subclass FormTest and fully implement.
+
+- End-to-end testing (with Selenium): This is partially implemented, and most of the necessary FormStorm functions have been abstracted. Just need to subclass FormTest and fully implement.
 - Tests for DRF Serializers. "SerializerStorm"
 - Set up CI
+- Handle the obscure, "long tail" cases by implementing a framework to define tests for any type of constraint (such as multi-column uniqueness constraints). To do this, a "sub-test" would define one or more form submissions and the (boolean) result expected. Sub-tests would be combined with each other and with the standard combinatorial mix of good/bad values so that all fields are tested simultaneously. A tentative definition of the sub-tests is below:
+
+
+    sub_tests = [
+        { # Sub-test 1
+            field_names=["field1","field2",..."fieldN"],
+            submissions = [ Each sub-test consists of multilpe submissions.
+                (  # Submission 1 
+                    value1,  # the value for field1
+                    value2,  # the value for field2
+                    ...
+                    valueN,  # the value for fieldN
+                    result,  # the expected result of the submission
+                ),
+                (...), # Submission 2
+                ...
+                (...)  # Submission N
+            ]
+        },
+        {...}, # Sub-test 2
+        ...
+        {...}  # Sub-test N
+    ]
+
+For example, suppose a model has two fields that have a multi-column uniqueness constraint:
+
+    class SomeModel(models.Model):
+        field1 = models.TextField()
+        field2 = models.TextField()
+        field3 = models.TextField()
+        ...
+        fieldN = models.TextField()
+
+        class Meta:
+            constraints = [
+                UniqueConstraint(
+                    fields=['field1', 'field2'], name='unique_together'
+                )
+            ] 
+
+The sub-test to test this constraint would be defined like this:
+
+    class SomeModelFormTest(FormTest):
+    	form = SomeModelForm
+        sub_tests = [
+            {
+                field_names=["field1","field2"],
+                submissions = [
+                    ("a","a", True),
+                    ("a","b", True),  # Duplicate values in one column are fine
+                    ("b","a", True),  # ... as are duplicates in the other column
+                    ("a","a", False)  # ... but the same values again should fail
+                ]
+            }
+        ]
+        field3 = FormElement(good=[...], bad=[...])
+        ...
+        fieldN = FormElement(good=[...], bad=[...])
+
+The advantage of this is that we can define a test only for the fields affected by a constraint, and have values for the other fields supplied by the normal good/bad value tests.
+

formstorm/FormElement.py

@@ -1,13 +1,13 @@
 from itertools import chain
 from django.db.models import Q
-from six import text_type
+# from six import text_type
 
 
 class FormElement(object):
     def __init__(self, good=[], bad=[], is_unique=False):
         self.bad = bad
         self.good = good
-        self.is_unique = is_unique 
+        self.is_unique = is_unique
 
     def build_iterator(self, form, field_name, is_e2e):
         """
@@ -26,19 +26,21 @@ def build_iterator(self, form, field_name, is_e2e):
         def _get_pk_for_q(q_object):
             # 1st: Find the model that the field points to.
             form_field = form._meta.model._meta.get_field(field_name)
-            try:  # Python 3
-                ref_model = form_field.remote_field.model
-            except AttributeError:
-                ref_model = form_field.rel.to
+            ref_model = form_field.remote_field.model
 
             # 2nd: Get the object that the Q object points to.
             ref_object = ref_model.objects.get(q_object)
 
             # 3rd: Get the identifier for that object.
-            if is_e2e:  # If we're doing an e2e test, reference by name.
-                return text_type(ref_object)
-            else:
-                return ref_object.pk
+
+            # If we're doing an e2e test, reference by name.
+            # (e2e is not implemented yet)
+            # if is_e2e:
+            #     return text_type(ref_object)
+            # else:
+            #    return ref_object.pk
+
+            return ref_object.pk
 
         def _replace_all_q(value_list):
             for i, g in enumerate(value_list):

formstorm/FormTest.py

@@ -24,7 +24,7 @@ def _build_elements(self, fields_to_ignore=[]):
                 continue
 
             # Filter out this class's FormElement properties
-            form_element = getattr(self, e) 
+            form_element = getattr(self, e)
             if type(form_element) is FormElement:
                 elements[e] = form_element.build_iterator(
                     is_e2e=self.is_e2e,
@@ -123,28 +123,24 @@ def run(self):
 
             if self._is_modelform:
                 if not has_run_uniqueness_test and form_is_good:
-                    # There's no way to verify that the uniqueness constraint 
+                    # There's no way to verify that the uniqueness constraint
                     # was the one that triggered the error. However, if the
                     # form was previously valid, and now it's not, and we've
                     # submitted the same input, then we can conclude that
                     # non-uniqueness was the issue.
                     self.submit_form(form_values)
                     assert not self.is_good()
+                    # Normally, we only test for valid/invalid, but let's
+                    # do a little extra and verify which fields caused errors.
                     for field in self.unique_elements:
                         assert self.bound_form.has_error(field)
                     has_run_uniqueness_test = True
 
                 transaction.savepoint_rollback(sid)
-        
+
         if should_run_uniqueness_test and not has_run_uniqueness_test:
             # If we make it to the end without having run the uniqueness test,
             # then it must be because no good input was specified.
             raise RuntimeError(
                 "Good input must be given to run uniqueness test."
             )
-
-    def run_uniqueness_tests(self):
-        pass
-
-    def run_individual_tests(self):
-        self._run(is_uniqueness_test=False)

tests/.coveragerc

@@ -1,7 +1,7 @@
 [run]
 branch = true
 omit = fstest/wsgi.py, manage.py
-source = .
+source = .,../formstorm
 
 [report]
 show_missing = true

tests/fstest/settings.py

@@ -36,7 +36,7 @@
     'django.contrib.admin', 'django.contrib.auth',
     'django.contrib.contenttypes', 'django.contrib.sessions',
     'django.contrib.messages', 'django.contrib.staticfiles', 'fstestapp',
-    'minimalapp'
+    'minimalapp', 'misctestsapp'
 ]
 
 MIDDLEWARE = [

tests/fstestapp/test.py

@@ -1,3 +1,4 @@
+import sys
 from formstorm import FormTest, FormElement
 from formstorm.iterhelpers import every_combo, dict_combo
 from .forms import BookForm
@@ -6,42 +7,78 @@
 from django.db.models import Q
 
 
+# class BookFormTest(FormTest):
+#     form = BookForm
+#     title = FormElement(
+#         good=["Moby Dick"],
+#         bad=[None, "", "A"*101],
+#         is_unique=True
+#     )
+#     subtitle = FormElement(
+#         good=[None, "", "or The Whale"],
+#         bad=["A"*101]
+#     )
+#     author = FormElement(
+#         good=[Q(name="Herman Melville")],
+#         bad=[None, "", -1]
+#     )
+#     is_fiction = FormElement(
+#         good=[True, False, None, "", -1, "A"],
+#         bad=[]  # Boolean input is either truthy or falsy, so nothing is bad.
+#     )
+#     pages = FormElement(
+#         good=[0, 10, 100],
+#         bad=[None, "", "A"]
+#     )
+#     genre = FormElement(
+#         good=every_combo([
+#             Q(name="Mystery"),
+#             Q(name="History"),
+#             Q(name="Humor")
+#         ]),
+#         bad=[None]
+#     )
+#     additional_values = [
+#         ({'title': "A"*100, 'subtitle': "A"*50}, True),
+#         ({'title': "A"*50, 'subtitle': "A"*100}, True),
+#         ({'title': "A"*100, 'subtitle': "A"*51}, False),
+#         ({'title': "A"*51, 'subtitle': "A"*100}, False),
+#     ]
+
+
 class BookFormTest(FormTest):
     form = BookForm
     title = FormElement(
         good=["Moby Dick"],
-        bad=[None, "", "A"*101],
+        bad=[None],
         is_unique=True
     )
     subtitle = FormElement(
-        good=[None, "", "or The Whale"],
+        good=[None],
         bad=["A"*101]
     )
     author = FormElement(
         good=[Q(name="Herman Melville")],
-        bad=[None, "", -1]
+        bad=[None]
     )
     is_fiction = FormElement(
-        good=[True, False, None, "", -1, "A"],
+        good=[True],
         bad=[]  # Boolean input is either truthy or falsy, so nothing is bad.
     )
     pages = FormElement(
-        good=[0, 10, 100],
-        bad=[None, "", "A"]
+        good=[100],
+        bad=[None]
     )
     genre = FormElement(
         good=every_combo([
             Q(name="Mystery"),
-            Q(name="History"),
             Q(name="Humor")
         ]),
         bad=[None]
     )
     additional_values = [
         ({'title': "A"*100, 'subtitle': "A"*50}, True),
-        ({'title': "A"*50, 'subtitle': "A"*100}, True),
         ({'title': "A"*100, 'subtitle': "A"*51}, False),
-        ({'title': "A"*51, 'subtitle': "A"*100}, False),
     ]
 
 
@@ -78,7 +115,14 @@ def test_dict_combo(self):
             {'a': 'C', 'b': 1, 'c': None},
         ]
 
-        assert list(x) == y
+        x = list(x)
+
+        # Compensate for non-deterministic dict insertion if using python <3.7.
+        if sys.version_info[:2] < (3, 7):
+            x.sort()
+            y.sort()
+
+        assert x == y
 
     def test_dict_combo_with_base_dict(self):
         x = dict_combo({
@@ -108,7 +152,14 @@ def test_dict_combo_with_base_dict(self):
             {'a': 'C', 'b': 1, 'c': None,  "d": "A", "e": 0, 'f': True},
         ]
 
-        assert list(x) == y
+        x = list(x)
+
+        # See note above.
+        if sys.version_info[:2] < (3, 7):
+            x.sort()
+            y.sort()
+
+        assert x == y
 
 
 class BookTestCase(TestCase):

tests/misctestsapp/forms.py

@@ -0,0 +1,13 @@
+from django.forms import ModelForm
+from minimalapp.models import Book
+from django import forms
+
+
+class NameForm(forms.Form):
+    your_name = forms.CharField(label='Your name', max_length=100)
+
+
+class BookForm(ModelForm):
+    class Meta:
+        model = Book
+        exclude = []

tests/misctestsapp/migrations/0001_initial.py

@@ -0,0 +1,21 @@
+# Generated by Django 3.0.5 on 2020-07-09 04:41
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Book',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('title', models.CharField(max_length=100)),
+            ],
+        ),
+    ]

tests/misctestsapp/models.py

@@ -0,0 +1,5 @@
+from django.db import models
+
+
+class Book(models.Model):
+    title = models.CharField(max_length=100, blank=False, null=False)