v1.0.0b1 release

.gitignore

@@ -1,3 +1,7 @@
+dist/
+formstorm.egg-info/
+tests/db.sqlite3
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

README.md

@@ -1,2 +1,142 @@
-# formstorm
-Brute-force form testing
+# FormStorm (v1.0.0 beta1)
+
+A library to test Django forms by trying (almost) every combination of valid and invalid input. - Sort of like a brute-force attack on your form.
+
+## Example:
+
+Suppose we have a form to create a book object. The book's name is mandatory,
+but the subtitle is optional. A `FormTest` is created that provides examples 
+of valid and invalid values for each field:
+
+
+    from django.forms import ModelForm
+    from formstorm import FormTest, FormElement
+    from django.test import TestCase
+
+
+    class Book(models.Model):
+        title = models.CharField(max_length=100, blank=False, null=False)
+        subtitle = models.CharField(max_length=100)
+
+
+    class BookForm(ModelForm):
+        class Meta:
+            model = Book
+            exclude = []
+
+
+    class BookFormTest(FormTest):
+    	form = BookForm
+    	title = FormElement(
+    		good = ["Moby Dick"],
+    		bad = [None, '', 'A'*101],
+    	)
+    	subtitle = FormElement(
+    		good = [None, "", "or The Whale"],
+    		bad = ["A"*101]
+    	)
+
+
+    class BookTestCase(TestCase):
+        def setUp(self):
+            self.theBookFormTest = BookFormTest()
+
+        def test_book_form(self):
+            self.theBookFormTest.run()
+
+
+When the `FormTest` runs, the form will be tested with every combination of 
+each field's possible values. Namely, the form will be tested with these values:
+
+
+|  title    | subtitle     | result  | 
+|-----------|--------------|---------| 
+| Moby Dick | None         | Valid   | 
+| None      | None         | Invalid | 
+| ""        | None         | Invalid | 
+| AA[...]AA | None         | Invalid | 
+| Moby Dick | ""           | Valid   | 
+| None      | ""           | Invalid | 
+| ""        | ""           | Invalid | 
+| AA[...]AA | ""           | Invalid | 
+| Moby Dick | or The Whale | Valid   | 
+| None      | or The Whale | Invalid | 
+| ""        | or The Whale | Invalid | 
+| AA[...]AA | or The Whale | Invalid | 
+| Moby Dick | AA[...]AA    | Invalid | 
+| None      | AA[...]AA    | Invalid | 
+| ""        | AA[...]AA    | Invalid | 
+| AA[...]AA | AA[...]AA    | Invalid | 
+
+Without something like FormStorm, you either have to tediously create test cases
+for each possible input value, or you have to just trust that the form behaves
+how you intend it to.
+
+## Advanced example:
+
+An example showing how to use different field types can be found in [tests/fstestapp/test.py](tests/fstestapp/test.py).
+
+Basically, all fields work as above, with the exception of ForeignKey and Many2Many fields whose values must be specified with `Q()` objects. Also, example values for multi-valued fields (such as Many2Many) can be created with the `every_combo()` function which returns every combination of the Many2Many options.
+
+## Install:
+
+    pip install formstorm
+
+## TODO:
+
+- Test to ensure that uniqueness constraints work. - Some provision for this feature has already been made, but it hasn't been fully implemented yet. 
+- End-to-end testing (with Selenium): This is partially implemented, and all of the necessary FormStorm functions have been abstracted. Just need to subclass FormTest and fully implement.
+- Tests for DRF Serializers. "SerializerStorm"
+- Set up CI
+- Support for dynamic values that depend on other values: For instance, suppose a Contact form either collects a company name or an individual name, depending on the contact type. We wouldn't want to supply a value for `individual_name` if it's not a company, so we'd need to implement like so:
+
+
+        class Contact(models.Model):
+            is_company = models.BooleanField(default=False)
+            company_name = models.CharField(max_length=100, blank=True, null=True)
+            individual_name = models.CharField(max_length=100, blank=True, null=True)
+        
+            def clean():
+                if self.is_company and not self.company_name:
+                    raise ValidationError("Please specify a company name.")
+                elif not self.is_company and not self.individual_name:
+                    raise ValidationError("Please provide your first and last name.")
+        
+        
+        class ContactFormTest(FormTest):
+            form = ContactForm
+            is_company = FormElement(good = [True, False], bad=[None])
+            company_name = FormElement(
+                good=["Acme, Inc."],
+                bad=[None],
+                only_if=["is_company"]
+            )
+            individual_name = FormElement(
+                good=["John Doe"],
+                bad=[None],
+                not_if=["is_company"]
+            )
+
+- Rather than specifying good/bad values, give the option to pass an iterator that returns (value, is_good).
+
+        class AuthorFormTest(FormTest):
+            form = AuthorForm
+            name = FormElement(
+                values=ValueHelper(values_iterator, depends_on=["field1","field2"])
+                good=[...],
+                bad=[...]
+            )
+- Support for [faker functions](https://github.com/joke2k/faker), using ValueHelper, as above. This would test the form with 3 names generated by fake.name():
+
+        from faker import Faker
+        fake = Faker()
+
+        class AuthorFormTest(FormTest):
+            form = AuthorForm
+            name = FormElement(
+                good=[
+                    "Herman Melville",
+                    "Charles Dickens",
+                    ValueHelper(fake.name, cardinality=3, is_good=True)
+                ],
+            )

formstorm/FormElement.py

@@ -0,0 +1,35 @@
+from itertools import chain
+from django.db.models import Q
+
+
+class FormElement(object):
+    def __init__(self,
+                 good=[],
+                 bad=[],
+                 values=[],
+                 only_if=[],
+                 not_if=[],
+                 unique=False,
+                 fk_field=None):
+        self.bad = bad
+        self.good = good
+        self.values = values
+        self.only_if = only_if
+        self.not_if = not_if
+
+    def build_iterator(self, ref_model, is_e2e):
+        for i, g in enumerate(self.good):
+            if type(g) is Q:
+                ref_object = ref_model.objects.get(g)
+                if is_e2e:  # If we're doing an e2e test, reference by name.
+                    self.good[i] = unicode(ref_object)
+                else:
+                    self.good[i] = ref_object.pk
+
+        self.iterator = chain(
+            [(x, True) for x in self.good],
+            [(x, False) for x in self.bad],
+            # self.values
+        )
+
+        return self.iterator

formstorm/FormTest.py

@@ -0,0 +1,70 @@
+from FormElement import FormElement
+from django.db import transaction
+from django.forms import ModelForm
+from iterhelpers import dict_combo
+
+
+class FormTest(object):
+    is_e2e = False
+
+    def is_good(self):
+        return self.bound_form.is_valid()
+
+    def submit_form(self, form_values):
+        self.bound_form = self.form(form_values)
+        if self._is_modelform and self.bound_form.is_valid():
+            print "Save!"
+            self.bound_form.save()
+
+    def _build_elements(self):
+        self.elements = {}
+        for e in dir(self):
+            # Filter out this class's FormElement properties
+            if type(getattr(self, e)) is FormElement:
+                # If this field is a fk/m2m, get the model that it points to.
+                try:
+                    ref_model = self.form._meta.model._meta.get_field(e).rel.to
+                except AttributeError:
+                    ref_model = None
+
+                self.elements[e] = getattr(self, e).build_iterator(
+                    is_e2e=self.is_e2e,
+                    ref_model=ref_model
+                )
+                getattr(self, e)
+
+    def __init__(self):
+        self._is_modelform = ModelForm in self.form.mro()
+        self._build_elements()
+        # Build iterable from the iterables of the sub-objects
+        self._iterator = dict_combo(self.elements)
+
+    def _run(self, is_uniqueness_test=False):
+        # i is a dictionary whose elements are tuples
+        # in the form (value, is_good)
+        for i in self._iterator:
+            # if any field is invalid, the form is invalid.
+            form_is_good = all([x[1][1] for x in i.items()])
+            form_values = {k: v[0] for k, v in i.items()}
+
+            if self._is_modelform and not is_uniqueness_test:
+                sid = transaction.savepoint()
+
+            self.submit_form(form_values)
+            assert self.is_good() == form_is_good
+            if is_uniqueness_test and form_is_good:
+                self.submit_form(form_values)
+                assert not self.is_good()
+
+            if self._is_modelform and not is_uniqueness_test:
+                transaction.savepoint_rollback(sid)
+
+    def run(self):
+        self._run(is_uniqueness_test=False)
+        # self._run(is_uniqueness_test=True)
+
+    def run_uniqueness_tests(self):
+        pass
+
+    def run_individual_tests(self):
+        self._run(is_uniqueness_test=False)

formstorm/__init__.py

@@ -0,0 +1,5 @@
+from FormTest import FormTest  # noqa: F401
+from FormElement import FormElement  # noqa: F401
+
+__version__ = '1.0.0beta1'
+name = "formstorm"

formstorm/iterhelpers.py

@@ -0,0 +1,62 @@
+import itertools
+
+
+def every_combo(items):
+    """
+        Given a list of n items, return every combination of length 1 .. n.
+
+        list(every_combo([1, 2, 3]))
+        [(1,), (2,), (3,), (1, 2), (1, 3), (2, 3), (1, 2, 3)]
+    """
+    return itertools.chain(*[
+        itertools.combinations(items, i)
+        for i in range(1, len(items) + 1)
+    ])
+
+
+def dict_combo(params):
+    """
+    params = {
+        "a": ["A","B","C"],
+        "b": [0, 1],
+        "c": [True, False, None],
+    }
+    for x in dict_combo(params):
+        print x
+
+    Returns:
+        {'a': 'A', 'c': True, 'b': 0}
+        {'a': 'A', 'c': True, 'b': 1}
+        {'a': 'A', 'c': False, 'b': 0}
+        {'a': 'A', 'c': False, 'b': 1}
+        {'a': 'A', 'c': None, 'b': 0}
+        {'a': 'A', 'c': None, 'b': 1}
+        {'a': 'B', 'c': True, 'b': 0}
+        {'a': 'B', 'c': True, 'b': 1}
+        {'a': 'B', 'c': False, 'b': 0}
+        {'a': 'B', 'c': False, 'b': 1}
+        {'a': 'B', 'c': None, 'b': 0}
+        {'a': 'B', 'c': None, 'b': 1}
+        {'a': 'C', 'c': True, 'b': 0}
+        {'a': 'C', 'c': True, 'b': 1}
+        {'a': 'C', 'c': False, 'b': 0}
+        {'a': 'C', 'c': False, 'b': 1}
+        {'a': 'C', 'c': None, 'b': 0}
+        {'a': 'C', 'c': None, 'b': 1}
+    """
+    # We don't want the key-->value paring to get mismatched.
+    # Convert a dictionary to (key, value) tuples,
+    # Then convert to a list of keys and a list of values
+    # The index of the key will be the index of the value.
+    keys, values = zip(*[
+        (key, value)
+        for key, value in params.items()
+    ])
+    # Take the cartesian product of all the sets.
+    # This will return an iterator with one item from each set,
+    # We take this an pack it back into a dictionary.
+    for y in itertools.product(*values):
+        out_dict = {}
+        for i in range(len(y)):
+            out_dict[keys[i]] = y[i]
+        yield out_dict

setup.py

@@ -0,0 +1,30 @@
+from setuptools import find_packages, setup
+from formstorm import __version__ as version_string
+
+setup(
+    name='formstorm',
+    version=version_string,
+    url='https://github.com/TravisDart/formstorm/',
+    description=(
+        'A tool to test Django forms by trying (almost) every '
+        'combination of valid and invalid input.'
+    ),
+    license='MIT',
+    author='Travis Dart',
+    author_email='[email protected]',
+    classifiers=[
+        'Framework :: Django',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: MIT License',
+        'Operating System :: OS Independent',
+        'Framework :: Django',
+        'Framework :: Django :: 1.11',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 2.7',
+        'Topic :: Software Development :: Testing',
+    ],
+    packages=find_packages(),
+    include_package_data=True,
+    # test_suite='nose.collector',
+    tests_require=['django'],
+)