diff --git a/.gitignore b/.gitignore index 894a44c..d5824f9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +dist/ +formstorm.egg-info/ +tests/db.sqlite3 + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/README.md b/README.md index 369ff9e..4a30f31 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,142 @@ -# formstorm -Brute-force form testing +# FormStorm (v1.0.0 beta1) + +A library to test Django forms by trying (almost) every combination of valid and invalid input. - Sort of like a brute-force attack on your form. + +## Example: + +Suppose we have a form to create a book object. The book's name is mandatory, +but the subtitle is optional. A `FormTest` is created that provides examples +of valid and invalid values for each field: + + + from django.forms import ModelForm + from formstorm import FormTest, FormElement + from django.test import TestCase + + + class Book(models.Model): + title = models.CharField(max_length=100, blank=False, null=False) + subtitle = models.CharField(max_length=100) + + + class BookForm(ModelForm): + class Meta: + model = Book + exclude = [] + + + class BookFormTest(FormTest): + form = BookForm + title = FormElement( + good = ["Moby Dick"], + bad = [None, '', 'A'*101], + ) + subtitle = FormElement( + good = [None, "", "or The Whale"], + bad = ["A"*101] + ) + + + class BookTestCase(TestCase): + def setUp(self): + self.theBookFormTest = BookFormTest() + + def test_book_form(self): + self.theBookFormTest.run() + + +When the `FormTest` runs, the form will be tested with every combination of +each field's possible values. Namely, the form will be tested with these values: + + +| title | subtitle | result | +|-----------|--------------|---------| +| Moby Dick | None | Valid | +| None | None | Invalid | +| "" | None | Invalid | +| AA[...]AA | None | Invalid | +| Moby Dick | "" | Valid | +| None | "" | Invalid | +| "" | "" | Invalid | +| AA[...]AA | "" | Invalid | +| Moby Dick | or The Whale | Valid | +| None | or The Whale | Invalid | +| "" | or The Whale | Invalid | +| AA[...]AA | or The Whale | Invalid | +| Moby Dick | AA[...]AA | Invalid | +| None | AA[...]AA | Invalid | +| "" | AA[...]AA | Invalid | +| AA[...]AA | AA[...]AA | Invalid | + +Without something like FormStorm, you either have to tediously create test cases +for each possible input value, or you have to just trust that the form behaves +how you intend it to. + +## Advanced example: + +An example showing how to use different field types can be found in [tests/fstestapp/test.py](tests/fstestapp/test.py). + +Basically, all fields work as above, with the exception of ForeignKey and Many2Many fields whose values must be specified with `Q()` objects. Also, example values for multi-valued fields (such as Many2Many) can be created with the `every_combo()` function which returns every combination of the Many2Many options. + +## Install: + + pip install formstorm + +## TODO: + +- Test to ensure that uniqueness constraints work. - Some provision for this feature has already been made, but it hasn't been fully implemented yet. +- End-to-end testing (with Selenium): This is partially implemented, and all of the necessary FormStorm functions have been abstracted. Just need to subclass FormTest and fully implement. +- Tests for DRF Serializers. "SerializerStorm" +- Set up CI +- Support for dynamic values that depend on other values: For instance, suppose a Contact form either collects a company name or an individual name, depending on the contact type. We wouldn't want to supply a value for `individual_name` if it's not a company, so we'd need to implement like so: + + + class Contact(models.Model): + is_company = models.BooleanField(default=False) + company_name = models.CharField(max_length=100, blank=True, null=True) + individual_name = models.CharField(max_length=100, blank=True, null=True) + + def clean(): + if self.is_company and not self.company_name: + raise ValidationError("Please specify a company name.") + elif not self.is_company and not self.individual_name: + raise ValidationError("Please provide your first and last name.") + + + class ContactFormTest(FormTest): + form = ContactForm + is_company = FormElement(good = [True, False], bad=[None]) + company_name = FormElement( + good=["Acme, Inc."], + bad=[None], + only_if=["is_company"] + ) + individual_name = FormElement( + good=["John Doe"], + bad=[None], + not_if=["is_company"] + ) + +- Rather than specifying good/bad values, give the option to pass an iterator that returns (value, is_good). + + class AuthorFormTest(FormTest): + form = AuthorForm + name = FormElement( + values=ValueHelper(values_iterator, depends_on=["field1","field2"]) + good=[...], + bad=[...] + ) +- Support for [faker functions](https://github.com/joke2k/faker), using ValueHelper, as above. This would test the form with 3 names generated by fake.name(): + + from faker import Faker + fake = Faker() + + class AuthorFormTest(FormTest): + form = AuthorForm + name = FormElement( + good=[ + "Herman Melville", + "Charles Dickens", + ValueHelper(fake.name, cardinality=3, is_good=True) + ], + ) diff --git a/formstorm/FormElement.py b/formstorm/FormElement.py new file mode 100644 index 0000000..abb7a8f --- /dev/null +++ b/formstorm/FormElement.py @@ -0,0 +1,35 @@ +from itertools import chain +from django.db.models import Q + + +class FormElement(object): + def __init__(self, + good=[], + bad=[], + values=[], + only_if=[], + not_if=[], + unique=False, + fk_field=None): + self.bad = bad + self.good = good + self.values = values + self.only_if = only_if + self.not_if = not_if + + def build_iterator(self, ref_model, is_e2e): + for i, g in enumerate(self.good): + if type(g) is Q: + ref_object = ref_model.objects.get(g) + if is_e2e: # If we're doing an e2e test, reference by name. + self.good[i] = unicode(ref_object) + else: + self.good[i] = ref_object.pk + + self.iterator = chain( + [(x, True) for x in self.good], + [(x, False) for x in self.bad], + # self.values + ) + + return self.iterator diff --git a/formstorm/FormTest.py b/formstorm/FormTest.py new file mode 100644 index 0000000..3cd73d4 --- /dev/null +++ b/formstorm/FormTest.py @@ -0,0 +1,70 @@ +from FormElement import FormElement +from django.db import transaction +from django.forms import ModelForm +from iterhelpers import dict_combo + + +class FormTest(object): + is_e2e = False + + def is_good(self): + return self.bound_form.is_valid() + + def submit_form(self, form_values): + self.bound_form = self.form(form_values) + if self._is_modelform and self.bound_form.is_valid(): + print "Save!" + self.bound_form.save() + + def _build_elements(self): + self.elements = {} + for e in dir(self): + # Filter out this class's FormElement properties + if type(getattr(self, e)) is FormElement: + # If this field is a fk/m2m, get the model that it points to. + try: + ref_model = self.form._meta.model._meta.get_field(e).rel.to + except AttributeError: + ref_model = None + + self.elements[e] = getattr(self, e).build_iterator( + is_e2e=self.is_e2e, + ref_model=ref_model + ) + getattr(self, e) + + def __init__(self): + self._is_modelform = ModelForm in self.form.mro() + self._build_elements() + # Build iterable from the iterables of the sub-objects + self._iterator = dict_combo(self.elements) + + def _run(self, is_uniqueness_test=False): + # i is a dictionary whose elements are tuples + # in the form (value, is_good) + for i in self._iterator: + # if any field is invalid, the form is invalid. + form_is_good = all([x[1][1] for x in i.items()]) + form_values = {k: v[0] for k, v in i.items()} + + if self._is_modelform and not is_uniqueness_test: + sid = transaction.savepoint() + + self.submit_form(form_values) + assert self.is_good() == form_is_good + if is_uniqueness_test and form_is_good: + self.submit_form(form_values) + assert not self.is_good() + + if self._is_modelform and not is_uniqueness_test: + transaction.savepoint_rollback(sid) + + def run(self): + self._run(is_uniqueness_test=False) + # self._run(is_uniqueness_test=True) + + def run_uniqueness_tests(self): + pass + + def run_individual_tests(self): + self._run(is_uniqueness_test=False) diff --git a/formstorm/__init__.py b/formstorm/__init__.py new file mode 100644 index 0000000..a05cbc2 --- /dev/null +++ b/formstorm/__init__.py @@ -0,0 +1,5 @@ +from FormTest import FormTest # noqa: F401 +from FormElement import FormElement # noqa: F401 + +__version__ = '1.0.0beta1' +name = "formstorm" diff --git a/formstorm/iterhelpers.py b/formstorm/iterhelpers.py new file mode 100644 index 0000000..dbb6199 --- /dev/null +++ b/formstorm/iterhelpers.py @@ -0,0 +1,62 @@ +import itertools + + +def every_combo(items): + """ + Given a list of n items, return every combination of length 1 .. n. + + list(every_combo([1, 2, 3])) + [(1,), (2,), (3,), (1, 2), (1, 3), (2, 3), (1, 2, 3)] + """ + return itertools.chain(*[ + itertools.combinations(items, i) + for i in range(1, len(items) + 1) + ]) + + +def dict_combo(params): + """ + params = { + "a": ["A","B","C"], + "b": [0, 1], + "c": [True, False, None], + } + for x in dict_combo(params): + print x + + Returns: + {'a': 'A', 'c': True, 'b': 0} + {'a': 'A', 'c': True, 'b': 1} + {'a': 'A', 'c': False, 'b': 0} + {'a': 'A', 'c': False, 'b': 1} + {'a': 'A', 'c': None, 'b': 0} + {'a': 'A', 'c': None, 'b': 1} + {'a': 'B', 'c': True, 'b': 0} + {'a': 'B', 'c': True, 'b': 1} + {'a': 'B', 'c': False, 'b': 0} + {'a': 'B', 'c': False, 'b': 1} + {'a': 'B', 'c': None, 'b': 0} + {'a': 'B', 'c': None, 'b': 1} + {'a': 'C', 'c': True, 'b': 0} + {'a': 'C', 'c': True, 'b': 1} + {'a': 'C', 'c': False, 'b': 0} + {'a': 'C', 'c': False, 'b': 1} + {'a': 'C', 'c': None, 'b': 0} + {'a': 'C', 'c': None, 'b': 1} + """ + # We don't want the key-->value paring to get mismatched. + # Convert a dictionary to (key, value) tuples, + # Then convert to a list of keys and a list of values + # The index of the key will be the index of the value. + keys, values = zip(*[ + (key, value) + for key, value in params.items() + ]) + # Take the cartesian product of all the sets. + # This will return an iterator with one item from each set, + # We take this an pack it back into a dictionary. + for y in itertools.product(*values): + out_dict = {} + for i in range(len(y)): + out_dict[keys[i]] = y[i] + yield out_dict diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..c7158b6 --- /dev/null +++ b/setup.py @@ -0,0 +1,30 @@ +from setuptools import find_packages, setup +from formstorm import __version__ as version_string + +setup( + name='formstorm', + version=version_string, + url='https://github.com/TravisDart/formstorm/', + description=( + 'A tool to test Django forms by trying (almost) every ' + 'combination of valid and invalid input.' + ), + license='MIT', + author='Travis Dart', + author_email='git@travisdart.com', + classifiers=[ + 'Framework :: Django', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Operating System :: OS Independent', + 'Framework :: Django', + 'Framework :: Django :: 1.11', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.7', + 'Topic :: Software Development :: Testing', + ], + packages=find_packages(), + include_package_data=True, + # test_suite='nose.collector', + tests_require=['django'], +) diff --git a/tests/fstest/__init__.py b/tests/fstest/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/fstest/settings.py b/tests/fstest/settings.py new file mode 100644 index 0000000..039f107 --- /dev/null +++ b/tests/fstest/settings.py @@ -0,0 +1,136 @@ +""" +Django settings for fstest project. + +Generated by 'django-admin startproject' using Django 2.0.2. + +For more information on this file, see +https://docs.djangoproject.com/en/2.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/2.0/ref/settings/ +""" + +import os +import sys + + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# Manually add the formstorm folder to the Python path. +sys.path.append(os.path.dirname(BASE_DIR)) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/2.0/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = '1zr1u$(z11q#3hziri&zdi-ahm8biko#&k4)+3xas*7=g22%_e' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + + 'fstestapp' +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'fstest.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'fstest.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/2.0/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + } +} + + +# Password validation +# https://docs.djangoproject.com/en/2.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': ( + 'django.contrib.auth.password_validation.' + 'UserAttributeSimilarityValidator' + ), + }, + { + 'NAME': ( + 'django.contrib.auth.password_validation.MinimumLengthValidator' + ), + }, + { + 'NAME': ( + 'django.contrib.auth.password_validation.CommonPasswordValidator' + ), + }, + { + 'NAME': ( + 'django.contrib.auth.password_validation.NumericPasswordValidator' + ), + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/2.0/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/2.0/howto/static-files/ + +STATIC_URL = '/static/' diff --git a/tests/fstest/urls.py b/tests/fstest/urls.py new file mode 100644 index 0000000..637600f --- /dev/null +++ b/tests/fstest/urls.py @@ -0,0 +1 @@ +urlpatterns = [] diff --git a/tests/fstest/wsgi.py b/tests/fstest/wsgi.py new file mode 100644 index 0000000..4c73ba1 --- /dev/null +++ b/tests/fstest/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for fstest project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/2.0/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "fstest.settings") + +application = get_wsgi_application() diff --git a/tests/fstestapp/__init__.py b/tests/fstestapp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/fstestapp/apps.py b/tests/fstestapp/apps.py new file mode 100644 index 0000000..d0bcc28 --- /dev/null +++ b/tests/fstestapp/apps.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.apps import AppConfig + + +class Test2Config(AppConfig): + name = 'fstestapp' diff --git a/tests/fstestapp/forms.py b/tests/fstestapp/forms.py new file mode 100644 index 0000000..cd07646 --- /dev/null +++ b/tests/fstestapp/forms.py @@ -0,0 +1,14 @@ +from django.forms import ModelForm +from .models import Author, Book + + +class AuthorForm(ModelForm): + class Meta: + model = Author + exclude = [] + + +class BookForm(ModelForm): + class Meta: + model = Book + exclude = [] diff --git a/tests/fstestapp/migrations/0001_initial.py b/tests/fstestapp/migrations/0001_initial.py new file mode 100644 index 0000000..f6c8783 --- /dev/null +++ b/tests/fstestapp/migrations/0001_initial.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-08-20 03:46 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Author', + fields=[ + ('id', models.AutoField(auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID')), + ('name', models.CharField(max_length=100)), + ], + ), + migrations.CreateModel( + name='Book', + fields=[ + ('id', models.AutoField(auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID')), + ('title', models.CharField(max_length=100, unique=True)), + ('subtitle', models.CharField(blank=True, + max_length=100, + null=True)), + ('is_fiction', models.BooleanField(default=False)), + ('pages', models.IntegerField(default=False)), + ('author', models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to='fstestapp.Author') + ), + ], + ), + migrations.CreateModel( + name='Genre', + fields=[ + ('id', models.AutoField(auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID')), + ('name', models.CharField(max_length=100)), + ], + ), + migrations.AddField( + model_name='book', + name='genre', + field=models.ManyToManyField(to='fstestapp.Genre'), + ), + ] diff --git a/tests/fstestapp/migrations/__init__.py b/tests/fstestapp/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/fstestapp/models.py b/tests/fstestapp/models.py new file mode 100644 index 0000000..97fc06d --- /dev/null +++ b/tests/fstestapp/models.py @@ -0,0 +1,27 @@ +from django.db import models + + +class Author(models.Model): + name = models.CharField(max_length=100) + + def __unicode__(self): + return self.name + + +class Genre(models.Model): + name = models.CharField(max_length=100) + + def __unicode__(self): + return self.name + + +class Book(models.Model): + title = models.CharField(max_length=100, unique=True) + subtitle = models.CharField(max_length=100, blank=True, null=True) + author = models.ForeignKey(Author) + is_fiction = models.BooleanField(default=False) + pages = models.IntegerField(default=False) + genre = models.ManyToManyField(Genre) + + def __unicode__(self): + return self.title diff --git a/tests/fstestapp/test.py b/tests/fstestapp/test.py new file mode 100644 index 0000000..9c6ceae --- /dev/null +++ b/tests/fstestapp/test.py @@ -0,0 +1,51 @@ +from formstorm import FormTest, FormElement +from formstorm.iterhelpers import every_combo +from forms import BookForm +from django.test import TestCase +from models import Author, Genre +from django.db.models import Q + + +class BookFormTest(FormTest): + form = BookForm + title = FormElement( + good=["Moby Dick"], + bad=[None, '', "A"*101], + ) + subtitle = FormElement( + good=[None, "", "or The Whale"], + bad=["A"*101] + ) + author = FormElement( + good=[Q(name="Herman Melville")], + bad=[None, "", -1] + ) + is_fiction = FormElement( + good=[True, False], + bad=[None, "", -1, "A"] + ) + pages = FormElement( + good=[0, 10, 100], + bad=[None, "", "A"] + ) + genre = FormElement( + good=every_combo([ + Q(name="Mystery"), + Q(name="History"), + Q(name="Humor") + ]), + bad=[None] + ) + + +class BookTestCase(TestCase): + def setUp(self): + Genre(name="Mystery").save() + Genre(name="History").save() + Genre(name="Humor").save() + Author(name="Herman Melville").save() + Author(name="Charles Dickens").save() + self.theBookFormTest = BookFormTest() + + def test_book_form(self): + self.theBookFormTest.run() diff --git a/tests/manage.py b/tests/manage.py new file mode 100755 index 0000000..a60314d --- /dev/null +++ b/tests/manage.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +import os +import sys + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "fstest.settings") + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) + execute_from_command_line(sys.argv)