diff --git a/.gitignore b/.gitignore index 289f8adcf..522b416bd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.settings *.pyc .DS_Store _build @@ -7,6 +8,8 @@ dist build MANIFEST .tox +env +env3 # Because I'm ghetto like that. tests/pyelasticsearch.py diff --git a/AUTHORS b/AUTHORS index d4f1a10b9..ec496ce47 100644 --- a/AUTHORS +++ b/AUTHORS @@ -67,3 +67,9 @@ Thanks to * Rodrigo Guzman (rz) for a fix to query handling in the ``simple`` backend. * Martin J. Laubach (mjl) for fixing the logic used when combining querysets * Eric Holscher (ericholscher) for a docs fix. + * Erik Rose (erikrose) for a quick pyelasticsearch-compatibility patch + * Stefan Wehrmeyer (stefanw) for a simple search filter fix + * Dan Watson (dcwatson) for various patches. + * Andrew Schoen (andrewschoen) for the addition of ``HAYSTACK_IDENTIFIER_METHOD`` + * Pablo SEMINARIO (pabluk) for a docs fix. + * Eric Thurgood (ethurgood) for a import fix in the Elasticssearch backend. diff --git a/README.rst b/README.rst index 2ad73c329..2665f0033 100644 --- a/README.rst +++ b/README.rst @@ -3,7 +3,7 @@ Haystack ======== :author: Daniel Lindsley -:date: 2012/03/30 +:date: 2013/04/28 Haystack provides modular search for Django. It features a unified, familiar API that allows you to plug in different search backends (such as Solr_, @@ -33,7 +33,8 @@ Documentation ============= * Development version: http://docs.haystacksearch.org/ -* v1.2.X: http://django-haystack.readthedocs.org/en/v1.2.6/ +* v2.0.X: http://django-haystack.readthedocs.org/en/v2.0.0/ +* v1.2.X: http://django-haystack.readthedocs.org/en/v1.2.7/ * v1.1.X: http://django-haystack.readthedocs.org/en/v1.1/ @@ -48,19 +49,3 @@ Haystack has a relatively easily-met set of requirements. Additionally, each backend has its own requirements. You should refer to http://docs.haystacksearch.org/dev/installing_search_engines.html for more details. - - -Commercial Support -================== - -If you're using Haystack in a commercial environment, paid support is available -from `Toast Driven`_. Services offered include: - -* Advice/help with setup -* Implementation in your project -* Bugfixes in Haystack itself -* Features in Haystack itself - -If you're interested, please contact Daniel Lindsley (daniel@toastdriven.com). - -.. _`Toast Driven`: http://toastdriven.com/ diff --git a/docs/autocomplete.rst b/docs/autocomplete.rst index fab06be4d..1872407c1 100644 --- a/docs/autocomplete.rst +++ b/docs/autocomplete.rst @@ -30,19 +30,19 @@ Example (continuing from the tutorial):: import datetime from haystack import indexes from myapp.models import Note - - + + class NoteIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=True) author = indexes.CharField(model_attr='user') pub_date = indexes.DateTimeField(model_attr='pub_date') # We add this for autocomplete. content_auto = indexes.EdgeNgramField(model_attr='content') - + def get_model(self): return Note - - def index_queryset(self): + + def index_queryset(self, using=None): """Used when the entire index for model is updated.""" return Note.objects.filter(pub_date__lte=datetime.datetime.now()) @@ -59,7 +59,7 @@ You simply provide a field & the query you wish to search on to the search would look like:: from haystack.query import SearchQuerySet - + SearchQuerySet().autocomplete(content_auto='old') # Result match things like 'goldfish', 'cuckold' & 'older'. @@ -70,8 +70,151 @@ If you need more control over your results, you can use standard ``SearchQuerySet.filter`` calls. For instance:: from haystack.query import SearchQuerySet - + sqs = SearchQuerySet().filter(content_auto=request.GET.get('q', '')) This can also be extended to use ``SQ`` for more complex queries (and is what's being done under the hood in the ``SearchQuerySet.autocomplete`` method). + + +Example Implementation +====================== + +The above is the low-level backend portion of how you implement autocomplete. +To make it work in browser, you need both a view to run the autocomplete +& some Javascript to fetch the results. + +Since it comes up often, here is an example implementation of those things. + +.. warning:: + + This code comes with no warranty. Don't ask for support on it. If you + copy-paste it & it burns down your server room, I'm not liable for any + of it. + + It worked this one time on my machine in a simulated environment. + + And yeah, semicolon-less + 2 space + comma-first. Deal with it. + +A stripped-down view might look like:: + + # views.py + import simplejson as json + from django.http import HttpResponse + from haystack.query import SearchQuerySet + + + def autocomplete(request): + sqs = SearchQuerySet().autocomplete(request.GET.get('q', ''))[:5] + suggestions = [result.title for result in sqs] + # Make sure you return a JSON object, not a bare list. + # Otherwise, you could be vulnerable to an XSS attack. + the_data = json.dumps({ + 'results': suggestions + }) + return HttpResponse(the_data, content_type='application/json') + +The template might look like:: + + + + + + Autocomplete Example + + +

Autocomplete Example

+ +
+ + +
+ + + + + diff --git a/docs/backend_support.rst b/docs/backend_support.rst index d84d35a35..d62c088ac 100644 --- a/docs/backend_support.rst +++ b/docs/backend_support.rst @@ -50,7 +50,7 @@ Elasticsearch * Stored (non-indexed) fields * Highlighting * Spatial search -* Requires: pyelasticsearch 0.2+ & Elasticsearch 0.17.7+ +* Requires: pyelasticsearch 0.4+ & Elasticsearch 0.17.7+ Whoosh ------ @@ -59,10 +59,11 @@ Whoosh * Full SearchQuerySet support * Automatic query building +* "More Like This" functionality * Term Boosting * Stored (non-indexed) fields * Highlighting -* Requires: whoosh (1.1.1+) +* Requires: whoosh (2.0.0+) Xapian ------ @@ -89,7 +90,7 @@ Backend Support Matrix +----------------+------------------------+---------------------+----------------+------------+----------+---------------+--------------+---------+ | Elasticsearch | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | +----------------+------------------------+---------------------+----------------+------------+----------+---------------+--------------+---------+ -| Whoosh | Yes | Yes | No | Yes | No | Yes | Yes | No | +| Whoosh | Yes | Yes | Yes | Yes | No | Yes | Yes | No | +----------------+------------------------+---------------------+----------------+------------+----------+---------------+--------------+---------+ | Xapian | Yes | Yes | Yes | Yes | Yes | Yes | Yes (plugin) | No | +----------------+------------------------+---------------------+----------------+------------+----------+---------------+--------------+---------+ diff --git a/docs/boost.rst b/docs/boost.rst index 3fae3fa49..26d0623be 100644 --- a/docs/boost.rst +++ b/docs/boost.rst @@ -23,14 +23,14 @@ Despite all being types of boost, they take place at different times and have slightly different effects on scoring. Term boost happens at query time (when the search query is run) and is based -around increasing the score is a certain word/phrase is seen. +around increasing the score if a certain word/phrase is seen. On the other hand, document & field boosts take place at indexing time (when the document is being added to the index). Document boost causes the relevance of the entire result to go up, where field boost causes only searches within that field to do better. -.. warning: +.. warning:: Be warned that boost is very, very sensitive & can hurt overall search quality if over-zealously applied. Even very small adjustments can affect @@ -47,7 +47,7 @@ Example:: # Slight increase in relevance for documents that include "banana". sqs = SearchQuerySet().boost('banana', 1.1) - + # Big decrease in relevance for documents that include "blueberry". sqs = SearchQuerySet().boost('blueberry', 0.8) @@ -63,16 +63,16 @@ Document boosting is done by adding a ``boost`` field to the prepared data from haystack import indexes from notes.models import Note - - + + class NoteSearchIndex(indexes.SearchIndex, indexes.Indexable): # Your regular fields here then... - + def prepare(self, obj): data = super(NoteSearchIndex, self).prepare(obj) data['boost'] = 1.1 return data - + Another approach might be to add a new field called ``boost``. However, this can skew your schema and is not encouraged. @@ -86,11 +86,11 @@ An example of this might be increasing the significance of a ``title``:: from haystack import indexes from notes.models import Note - - + + class NoteSearchIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=True) title = indexes.CharField(model_attr='title', boost=1.125) - + def get_model(self): return Note diff --git a/docs/conf.py b/docs/conf.py index 22ac9c1c2..d1deeb8cb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -45,9 +45,9 @@ # built documents. # # The short X.Y version. -version = '2.0' +version = '2.0.1' # The full version, including alpha/beta/rc tags. -release = '2.0.0-beta' +release = '2.0.1-dev' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/debugging.rst b/docs/debugging.rst index 648a677c6..3d5a8677a 100644 --- a/docs/debugging.rst +++ b/docs/debugging.rst @@ -38,11 +38,11 @@ Several issues can cause no results to be found. Most commonly it is either not running a ``rebuild_index`` to populate your index or having a blank ``document=True`` field, resulting in no content for the engine to search on. -* Do you have a ``search_sites.py`` that runs ``haystack.autodiscover``? -* Have you registered your models with the main ``haystack.site`` (usually - within your ``search_indexes.py``)? +* Do you have a ``search_indexes.py`` located within an installed app? * Do you have data in your database? * Have you run a ``./manage.py rebuild_index`` to index all of your content? +* Try running ``./manage.py rebuild_index -v2`` for more verbose output to + ensure data is being processed/inserted. * Start a Django shell (``./manage.py shell``) and try:: >>> from haystack.query import SearchQuerySet diff --git a/docs/faceting.rst b/docs/faceting.rst index cfe1a8e28..13f07c725 100644 --- a/docs/faceting.rst +++ b/docs/faceting.rst @@ -8,7 +8,7 @@ What Is Faceting? ----------------- Faceting is a way to provide users with feedback about the number of documents -which match terms they may be interested in. At it's simplest, it gives +which match terms they may be interested in. At its simplest, it gives document counts based on words in the corpus, date ranges, numeric ranges or even advanced queries. diff --git a/docs/glossary.rst b/docs/glossary.rst index d362fbe96..f6a1e6ee4 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -4,7 +4,7 @@ Glossary ======== -Search is a domain full of it's own jargon and definitions. As this may be an +Search is a domain full of its own jargon and definitions. As this may be an unfamiliar territory to many developers, what follows are some commonly used terms and what they mean. diff --git a/docs/index.rst b/docs/index.rst index 25cc44d38..3e2c41e21 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,8 +13,8 @@ Elasticsearch_, Whoosh_, Xapian_, etc.) without having to modify your code. .. note:: - This documentation represents the development version of Haystack. For - old versions of the documentation: `1.2`_, `1.1`_. + This documentation represents the development version of Haystack (2.0.x). + For old versions of the documentation: `1.2`_, `1.1`_. .. _`1.2`: http://django-haystack.readthedocs.org/en/v1.2.6/index.html .. _`1.1`: http://django-haystack.readthedocs.org/en/v1.1/index.html diff --git a/docs/inputtypes.rst b/docs/inputtypes.rst index 0c79ccd07..fe839e6cd 100644 --- a/docs/inputtypes.rst +++ b/docs/inputtypes.rst @@ -125,7 +125,7 @@ Example:: .. class:: haystack.inputs.AltParser -``AltParser`` let's you specify that a portion of the query should use a +``AltParser`` lets you specify that a portion of the query should use a separate parser in the search engine. This is search-engine-specific, so it may decrease the portability of your app. @@ -153,7 +153,7 @@ The ``prepare`` method lets you alter the query the user provided before it becomes of the main query. It is lazy, called as late as possible, right before the final query is built & shipped to the engine. -A full, if somewhat silly, example look like:: +A full, if somewhat silly, example looks like:: from haystack.inputs import Clean diff --git a/docs/installing_search_engines.rst b/docs/installing_search_engines.rst index f6d7c6054..25fee9524 100644 --- a/docs/installing_search_engines.rst +++ b/docs/installing_search_engines.rst @@ -42,7 +42,7 @@ somewhere on your ``PYTHONPATH``. .. note:: - ``pysolr`` has it's own dependencies that aren't covered by Haystack. For + ``pysolr`` has its own dependencies that aren't covered by Haystack. For best results, you should have an ElementTree variant install (preferably the ``lxml`` variant), ``httplib2`` for timeouts (though it will fall back to ``httplib``) and either the ``json`` module that comes with Python 2.5+ or @@ -74,7 +74,7 @@ Something like the following is suggested:: suggestions = indexes.FacetCharField() def prepare(self, obj): - prepared_data = super(NoteIndex, self).prepare(obj) + prepared_data = super(MySearchIndex, self).prepare(obj) prepared_data['suggestions'] = prepared_data['text'] return prepared_data @@ -117,8 +117,8 @@ Official Download Location: http://www.elasticsearch.org/download/ Elasticsearch is Java but comes in a pre-packaged form that requires very little other than the JRE. It's also very performant, scales easily and has -an advanced featureset. Haystack requires at least version 0.17.7 (0.18.6 is -current as of writing). Installation is best done using a package manager:: +an advanced featureset. Haystack requires at least version 0.90.0+. +Installation is best done using a package manager:: # On Mac OS X... brew install elasticsearch @@ -130,7 +130,7 @@ current as of writing). Installation is best done using a package manager:: elasticsearch -f -D es.config= # Example: - elasticsearch -f -D es.config=/usr/local/Cellar/elasticsearch/0.17.7/config/elasticsearch.yml + elasticsearch -f -D es.config=/usr/local/Cellar/elasticsearch/0.90.0/config/elasticsearch.yml You may have to alter the configuration to run on ``localhost`` when developing locally. Modifications should be done in a YAML file, the stock one being @@ -160,7 +160,7 @@ You'll also need an Elasticsearch binding: pyelasticsearch_ (**NOT** .. note:: - ``pyelasticsearch`` has it's own dependencies that aren't covered by + ``pyelasticsearch`` has its own dependencies that aren't covered by Haystack. You'll also need ``requests`` & ``simplejson`` for speedier JSON construction/parsing. @@ -172,11 +172,8 @@ Official Download Location: http://bitbucket.org/mchaput/whoosh/ Whoosh is pure Python, so it's a great option for getting started quickly and for development, though it does work for small scale live deployments. The -current recommended version is 1.3.1+. You can install via PyPI_ using:: - - sudo easy_install whoosh - # ... or ... - sudo pip install whoosh +current recommended version is 1.3.1+. You can install via PyPI_ using +``sudo easy_install whoosh`` or ``sudo pip install whoosh``. Note that, while capable otherwise, the Whoosh backend does not currently support "More Like This" or faceting. Support for these features has recently diff --git a/docs/management_commands.rst b/docs/management_commands.rst index 7ae1ecd96..bf9c9ffb9 100644 --- a/docs/management_commands.rst +++ b/docs/management_commands.rst @@ -164,7 +164,7 @@ following arguments:: If provided, determines which connection should be used. Default is ``default``. -.. warning: +.. warning:: This command does NOT update the ``schema.xml`` file for you. You either have to specify a ``filename`` flag or have to diff --git a/docs/migration_from_1_to_2.rst b/docs/migration_from_1_to_2.rst index 1e019915a..6159e06fe 100644 --- a/docs/migration_from_1_to_2.rst +++ b/docs/migration_from_1_to_2.rst @@ -155,7 +155,7 @@ A converted Haystack 2.X index should look like:: def get_model(self): return Note - def index_queryset(self): + def index_queryset(self, using=None): """Used when the entire index for model is updated.""" return self.get_model().objects.filter(pub_date__lte=datetime.datetime.now()) diff --git a/docs/multiple_index.rst b/docs/multiple_index.rst index 7bacf3862..c51b7341b 100644 --- a/docs/multiple_index.rst +++ b/docs/multiple_index.rst @@ -163,3 +163,39 @@ via the ``SearchQuerySet.using`` method:: Note that the models a ``SearchQuerySet`` is trying to pull from must all come from the same index. Haystack is not able to combine search queries against different indexes. + + +Custom Index Selection +====================== + +If a specific backend has been selected, the ``SearchIndex.index_queryset`` and +``SearchIndex.read_queryset`` will receive the backend name, giving indexes the +opportunity to customize the returned queryset. + +For example, a site which uses separate indexes for recent items and older +content might define ``index_queryset`` to filter the items based on date:: + + def index_queryset(self, using=None): + qs = Note.objects.all() + archive_limit = datetime.datetime.now() - datetime.timedelta(days=90) + + if using == "archive": + return qs.filter(pub_date__lte=archive_limit) + else: + return qs.filter(pub_date__gte=archive_limit) + + +Multi-lingual Content +--------------------- + +Most search engines require you to set the language at the index level. For +example, a multi-lingual site using Solr can use `multiple cores `_ and corresponding Haystack +backends using the language name. Under this scenario, queries are simple:: + + sqs = SearchQuerySet.using(lang).auto_query(…) + +During index updates, the Index's ``index_queryset`` method will need to filter +the items to avoid sending the wrong content to the search engine:: + + def index_queryset(self, using=None): + return Post.objects.filter(language=using) diff --git a/docs/other_apps.rst b/docs/other_apps.rst index 6713db572..e9751ff32 100644 --- a/docs/other_apps.rst +++ b/docs/other_apps.rst @@ -28,6 +28,14 @@ Also provides a queue-based setup, this time centered around Celery. Useful for keeping the index fresh per model instance or with the included task to call the ``update_index`` management command instead. +haystack-rqueue +--------------- + +https://github.com/mandx/haystack-rqueue (2.X compatible) + +Also provides a queue-based setup, this time centered around RQ. Useful +for keeping the index fresh using ``./manage.py rqworker``. + django-celery-haystack ---------------------- diff --git a/docs/running_tests.rst b/docs/running_tests.rst index b9f48e792..74bc7065a 100644 --- a/docs/running_tests.rst +++ b/docs/running_tests.rst @@ -7,11 +7,30 @@ Running Tests Dependencies ============ -``Haystack`` uses the `Mock `_ library for -testing. You will need to install it before running the tests:: +Everything +---------- + +The simplest way to get up and running with Haystack's tests is to run:: + + pip install -r tests/requirements.txt + +This installs all of the backend libraries & all dependencies for getting the +tests going. You will still have to setup search servers (for running Solr +tests, the spatial Solr tests & the Elasticsearch tests). + + +Cherry-Picked +------------- + +If you'd rather not run all the tests, install only the backends you need. +Additionally, ``Haystack`` uses the Mock_ library for testing. You will need +to install it before running the tests:: pip install mock +.. _Mock: http://pypi.python.org/pypi/mock + + Core Haystack Functionality =========================== diff --git a/docs/searchbackend_api.rst b/docs/searchbackend_api.rst index 9dc9b4462..d077fbf5d 100644 --- a/docs/searchbackend_api.rst +++ b/docs/searchbackend_api.rst @@ -58,7 +58,7 @@ specific to each one. .. method:: SearchBackend.search(self, query_string, sort_by=None, start_offset=0, end_offset=None, fields='', highlight=False, facets=None, date_facets=None, query_facets=None, narrow_queries=None, spelling_query=None, limit_to_registered_models=None, result_class=None, **kwargs) -Takes a query to search on and returns dictionary. +Takes a query to search on and returns a dictionary. The query should be a string that is appropriate syntax for the backend. diff --git a/docs/searchfield_api.rst b/docs/searchfield_api.rst index 8c037aefd..bf8466bd9 100644 --- a/docs/searchfield_api.rst +++ b/docs/searchfield_api.rst @@ -6,7 +6,7 @@ .. class:: SearchField -The ``SearchField`` and it's subclasses provides a way to declare what data +The ``SearchField`` and its subclasses provides a way to declare what data you're interested in indexing. They are used with ``SearchIndexes``, much like ``forms.*Field`` are used within forms or ``models.*Field`` within models. @@ -108,7 +108,7 @@ to be the primary field for searching within. Default is ``False``. .. attribute:: SearchField.indexed -A boolean flag for indicating whether or not the the data from this field will +A boolean flag for indicating whether or not the data from this field will be searchable within the index. Default is ``True``. The companion of this option is ``stored``. @@ -162,7 +162,7 @@ not to contain any data. Default is ``False``. Unlike Django's database layer, which injects a ``NULL`` into the database when a field is marked nullable, ``null=True`` will actually exclude that - field from being included with the document. This more efficient for the + field from being included with the document. This is more efficient for the search engine to deal with. ``stored`` diff --git a/docs/searchindex_api.rst b/docs/searchindex_api.rst index 949391fce..62dc22d52 100644 --- a/docs/searchindex_api.rst +++ b/docs/searchindex_api.rst @@ -34,7 +34,7 @@ For the impatient:: def get_model(self): return Note - def index_queryset(self): + def index_queryset(self, using=None): "Used when the entire index for model is updated." return self.get_model().objects.filter(pub_date__lte=datetime.datetime.now()) @@ -107,7 +107,7 @@ However, this can also hit the database quite heavily (think ``.get(pk=result.id)`` per object). If your search is popular, this can lead to a big performance hit. There are two ways to prevent this. The first way is ``SearchQuerySet.load_all``, which tries to group all similar objects and pull -them though one query instead of many. This still hits the DB and incurs a +them through one query instead of many. This still hits the DB and incurs a performance penalty. The other option is to leverage stored fields. By default, all fields in @@ -159,7 +159,7 @@ see, the churn rate of your data and what concerns are important to you The conventional method is to use ``SearchIndex`` in combination with cron jobs. Running a ``./manage.py update_index`` every couple hours will keep your data in sync within that timeframe and will handle the updates in a very -efficient batch. Additionally, Whoosh (and to a lesser extent Xapian) behave +efficient batch. Additionally, Whoosh (and to a lesser extent Xapian) behaves better when using this approach. Another option is to use ``RealtimeSignalProcessor``, which uses Django's @@ -236,7 +236,7 @@ you might write the following code:: return "%s <%s>" % (obj.user.get_full_name(), obj.user.email) This method should return a single value (or list/tuple/dict) to populate that -fields data upon indexing. Note that this method takes priority over whatever +field's data upon indexing. Note that this method takes priority over whatever data may come from the field itself. Just like ``Form.clean_FOO``, the field's ``prepare`` runs before the @@ -386,7 +386,7 @@ This method is required & you must override it to return the correct class. ``index_queryset`` ------------------ -.. method:: SearchIndex.index_queryset(self) +.. method:: SearchIndex.index_queryset(self, using=None) Get the default QuerySet to index when doing a full update. @@ -395,7 +395,7 @@ Subclasses can override this method to avoid indexing certain objects. ``read_queryset`` ----------------- -.. method:: SearchIndex.read_queryset(self) +.. method:: SearchIndex.read_queryset(self, using=None) Get the default QuerySet for read actions. @@ -526,7 +526,7 @@ with ``RelatedSearchQuerySet.load_all``. This is useful for post-processing the results from the query, enabling things like adding ``select_related`` or filtering certain data. -.. warning: +.. warning:: Utilizing this functionality can have negative performance implications. Please see the section on ``RelatedSearchQuerySet`` within @@ -609,7 +609,7 @@ For the impatient:: fields = ['user', 'pub_date'] # Note that regular ``SearchIndex`` methods apply. - def index_queryset(self): + def index_queryset(self, using=None): "Used when the entire index for model is updated." return Note.objects.filter(pub_date__lte=datetime.datetime.now()) diff --git a/docs/searchquery_api.rst b/docs/searchquery_api.rst index 63622af9e..305557e06 100644 --- a/docs/searchquery_api.rst +++ b/docs/searchquery_api.rst @@ -8,7 +8,7 @@ The ``SearchQuery`` class acts as an intermediary between ``SearchQuerySet``'s abstraction and ``SearchBackend``'s actual search. Given the metadata provided -by ``SearchQuerySet``, ``SearchQuery`` build the actual query and interacts +by ``SearchQuerySet``, ``SearchQuery`` builds the actual query and interacts with the ``SearchBackend`` on ``SearchQuerySet``'s behalf. This class must be at least partially implemented on a per-backend basis, as portions @@ -236,8 +236,8 @@ Adds a boosted term and the amount to boost it to the query. Runs a raw query (no parsing) against the backend. -This method causes the SearchQuery to ignore the standard query -generating facilities, running only what was provided instead. +This method causes the ``SearchQuery`` to ignore the standard query-generating +facilities, running only what was provided instead. Note that any kwargs passed along will override anything provided to the rest of the ``SearchQuerySet``. @@ -250,6 +250,12 @@ to the rest of the ``SearchQuerySet``. Allows backends with support for "More Like This" to return results similar to the provided instance. +``add_stats_query`` +~~~~~~~~~~~~~~~~~~~ +.. method:: SearchQuery.add_stats_query(self,stats_field,stats_facets) + +Adds stats and stats_facets queries for the Solr backend. + ``add_highlight`` ~~~~~~~~~~~~~~~~~ @@ -282,7 +288,7 @@ point passed in. ``add_field_facet`` ~~~~~~~~~~~~~~~~~~~ -.. method:: SearchQuery.add_field_facet(self, field) +.. method:: SearchQuery.add_field_facet(self, field, **options) Adds a regular facet on a field. diff --git a/docs/searchqueryset_api.rst b/docs/searchqueryset_api.rst index 705fa84c4..5fb039b39 100644 --- a/docs/searchqueryset_api.rst +++ b/docs/searchqueryset_api.rst @@ -1,4 +1,4 @@ -_ref-searchqueryset-api: +.. _ref-searchqueryset-api: ====================== ``SearchQuerySet`` API @@ -277,10 +277,18 @@ Example:: ``facet`` ~~~~~~~~~ -.. method:: SearchQuerySet.facet(self, field) +.. method:: SearchQuerySet.facet(self, field, **options) Adds faceting to a query for the provided field. You provide the field (from one -of the ``SearchIndex`` classes) you like to facet on. +of the ``SearchIndex`` classes) you like to facet on. Any keyword options you +provide will be passed along to the backend for that facet. + +Example:: + + # For SOLR (setting f.author.facet.*; see http://wiki.apache.org/solr/SimpleFacetParameters#Parameters) + SearchQuerySet().facet('author', mincount=1, limit=10) + # For ElasticSearch (see http://www.elasticsearch.org/guide/reference/api/search/facets/terms-facet.html) + SearchQuerySet().facet('author', size=10, order='term') In the search results you get back, facet counts will be populated in the ``SearchResult`` object. You can access them via the ``facet_counts`` method. @@ -303,7 +311,7 @@ amount of time between gaps as ``gap_by`` (one of ``'year'``, ``'month'``, You can also optionally provide a ``gap_amount`` to specify a different increment than ``1``. For example, specifying gaps by week (every seven days) -would would be ``gap_by='day', gap_amount=7``). +would be ``gap_by='day', gap_amount=7``). In the search results you get back, facet counts will be populated in the ``SearchResult`` object. You can access them via the ``facet_counts`` method. @@ -352,9 +360,40 @@ Spatial: Adds a distance-based search to the query. See the :ref:`ref-spatial` docs for more information. +``stats`` +~~~~~~~~~ + +.. method:: SearchQuerySet.stats(self, field): + +Adds stats to a query for the provided field. This is supported on +Solr only. You provide the field (from one of the ``SearchIndex`` +classes) you would like stats on. + +In the search results you get back, stats will be populated in the +``SearchResult`` object. You can access them via the `` stats_results`` method. + +Example:: + + # Get stats on the author field. + SearchQuerySet().filter(content='foo').stats('author') + +``stats_facet`` +~~~~~~~~~~~~~~~ +.. method:: SearchQuerySet.stats_facet(self, field, +.. facet_fields=None): + +Adds stats facet for the given field and facet_fields represents the +faceted fields. This is supported on Solr only. + +Example:: + + # Get stats on the author field, and stats on the author field + faceted by bookstore. + SearchQuerySet().filter(content='foo').stats_facet('author','bookstore') + + ``distance`` ~~~~~~~~~~~~ - .. method:: SearchQuerySet.distance(self, field, point): Spatial: Denotes results must have distance measurements from the @@ -505,7 +544,7 @@ for similar results. The instance you pass in should be an indexed object. Previously called methods will have an effect on the provided results. It will evaluate its own backend-specific query and populate the -`SearchQuerySet`` in the same manner as other methods. +``SearchQuerySet`` in the same manner as other methods. Example:: @@ -624,6 +663,52 @@ Example:: # 'queries': {} # } +``stats_results`` +~~~~~~~~~~~~~~~~~ + +.. method:: SearchQuerySet.stats_results(self): + +Returns the stats results found by the query. + + This will cause the query to +execute and should generally be used when presenting the data (template-level). + +You receive back a dictionary with three keys: ``fields``, ``dates`` and +``queries``. Each contains the facet counts for whatever facets you specified +within your ``SearchQuerySet``. + +.. note:: + + The resulting dictionary may change before 1.0 release. It's fairly + backend-specific at the time of writing. Standardizing is waiting on + implementing other backends that support faceting and ensuring that the + results presented will meet their needs as well. + +Example:: + + # Count document hits for each author. + sqs = SearchQuerySet().filter(content='foo').stats('price') + + sqs.stats_results() + + # Gives the following response + # { + # 'stats_fields':{ + # 'author:{ + # 'min': 0.0, + # 'max': 2199.0, + # 'sum': 5251.2699999999995, + # 'count': 15, + # 'missing': 11, + # 'sumOfSquares': 6038619.160300001, + # 'mean': 350.08466666666664, + # 'stddev': 547.737557906113 + # } + # } + # + # } + + ``spelling_suggestion`` ~~~~~~~~~~~~~~~~~~~~~~~ @@ -632,7 +717,8 @@ Example:: Returns the spelling suggestion found by the query. To work, you must set ``INCLUDE_SPELLING`` within your connection's -settings dictionary to ``True``. Otherwise, ``None`` will be returned. +settings dictionary to ``True``, and you must rebuild your index afterwards. +Otherwise, ``None`` will be returned. This method causes the query to evaluate and run the search if it hasn't already run. Search results will be populated as normal but with an additional spelling diff --git a/docs/settings.rst b/docs/settings.rst index 329ca4a94..c0538d3bf 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -93,8 +93,8 @@ The following options are optional: * ``INCLUDE_SPELLING`` - Include spelling suggestions. Default is ``False`` * ``BATCH_SIZE`` - How many records should be updated at once via the management commands. Default is ``1000``. -* ``TIMEOUT`` - (Solr-only) How long to wait (in seconds) before the connection - times out. Default is ``10``. +* ``TIMEOUT`` - (Solr and ElasticSearch) How long to wait (in seconds) before + the connection times out. Default is ``10``. * ``STORAGE`` - (Whoosh-only) Which storage engine to use. Accepts ``file`` or ``ram``. Default is ``file``. * ``POST_LIMIT`` - (Whoosh-only) How large the file sizes can be. Default is @@ -268,3 +268,20 @@ An example:: HAYSTACK_DJANGO_ID_FIELD = 'my_django_id' Default is ``django_id``. + + +``HAYSTACK_IDENTIFIER_METHOD`` +============================== + +**Optional** + +This setting allows you to provide a custom method for +``haystack.utils.get_identifier``. Useful when the default identifier +pattern of .. isn't suited to your +needs. + +An example:: + + HAYSTACK_IDENTIFIER_METHOD = 'my_app.module.get_identifier' + +Default is ``haystack.utils.default_get_identifier``. diff --git a/docs/signal_processors.rst b/docs/signal_processors.rst index d79e671ec..a3fbc921e 100644 --- a/docs/signal_processors.rst +++ b/docs/signal_processors.rst @@ -24,7 +24,7 @@ has been introduced. In it's simplest form, the ``SignalProcessor`` listens to whatever signals are setup & can be configured to then trigger the updates without having to change any ``SearchIndex`` code. -.. warning: +.. warning:: Incorporating Haystack's ``SignalProcessor`` into your setup **will** increase the overall load (CPU & perhaps I/O depending on configuration). @@ -74,7 +74,7 @@ Configuration looks like:: This causes **all** ``SearchIndex`` classes to work in a realtime fashion. -.. note: +.. note:: These updates happen in-process, which if a request-response cycle is involved, may cause the user with the browser to sit & wait for indexing to diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 65dc0a0fd..0452f81ea 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -27,8 +27,14 @@ which ever search engine you choose. If you hit a stumbling block, there is both a `mailing list`_ and `#haystack on irc.freenode.net`_ to get help. +.. note:: + + You can participate in and/or track the development of Haystack by + subscribing to the `development mailing list`_. + .. _mailing list: http://groups.google.com/group/django-haystack .. _#haystack on irc.freenode.net: irc://irc.freenode.net/haystack +.. _development mailing list: http://groups.google.com/group/django-haystack-dev This tutorial assumes that you have a basic familiarity with the various major parts of Django (models/forms/views/settings/URLconfs) and tailored to the @@ -57,6 +63,16 @@ backend to get started. There is a quick-start guide to official instructions. +Installation +============= + +Use your favorite Python package manager to install the app from PyPI, e.g. + +Example:: + + pip install django-haystack + + Configuration ============= @@ -156,7 +172,7 @@ Example:: import os HAYSTACK_CONNECTIONS = { 'default': { - 'ENGINE': 'haystack.backends.xapian_backend.XapianEngine', + 'ENGINE': 'xapian_backend.XapianEngine', 'PATH': os.path.join(os.path.dirname(__file__), 'xapian_index'), }, } @@ -221,7 +237,7 @@ Haystack to automatically pick it up. The ``NoteIndex`` should look like:: def get_model(self): return Note - def index_queryset(self): + def index_queryset(self, using=None): """Used when the entire index for model is updated.""" return self.get_model().objects.filter(pub_date__lte=datetime.datetime.now()) diff --git a/docs/utils.rst b/docs/utils.rst index 57919581b..7d42fc597 100644 --- a/docs/utils.rst +++ b/docs/utils.rst @@ -12,7 +12,7 @@ Included here are some of the general use bits included with Haystack. .. function:: get_identifier(obj_or_string) -Get an unique identifier for the object or a string representing the +Gets an unique identifier for the object or a string representing the object. -If not overridden, uses ... +If not overridden, uses ``..``. diff --git a/docs/views_and_forms.rst b/docs/views_and_forms.rst index 9b895ebe5..8f0c1be39 100644 --- a/docs/views_and_forms.rst +++ b/docs/views_and_forms.rst @@ -87,24 +87,27 @@ associated with it. You might create a form that looked as follows:: from django import forms from haystack.forms import SearchForm - - + + class DateRangeSearchForm(SearchForm): start_date = forms.DateField(required=False) end_date = forms.DateField(required=False) - + def search(self): # First, store the SearchQuerySet received from other processing. sqs = super(DateRangeSearchForm, self).search() - + + if not self.is_valid(): + return self.no_query_found() + # Check to see if a start_date was chosen. if self.cleaned_data['start_date']: sqs = sqs.filter(pub_date__gte=self.cleaned_data['start_date']) - + # Check to see if an end_date was chosen. if self.cleaned_data['end_date']: sqs = sqs.filter(pub_date__lte=self.cleaned_data['end_date']) - + return sqs This form adds two new fields for (optionally) choosing the start and end dates. @@ -150,9 +153,9 @@ URLconf should look something like:: from haystack.forms import ModelSearchForm from haystack.query import SearchQuerySet from haystack.views import SearchView - + sqs = SearchQuerySet().filter(author='john') - + # Without threading... urlpatterns = patterns('haystack.views', url(r'^$', SearchView( @@ -161,10 +164,10 @@ URLconf should look something like:: form_class=SearchForm ), name='haystack_search'), ) - + # With threading... from haystack.views import SearchView, search_view_factory - + urlpatterns = patterns('haystack.views', url(r'^$', search_view_factory( view_class=SearchView, @@ -267,17 +270,14 @@ As with the forms, inheritance is likely your best bet. In this case, the ``SearchView``. The complete code for the ``FacetedSearchView`` looks like:: class FacetedSearchView(SearchView): - def __name__(self): - return "FacetedSearchView" - def extra_context(self): extra = super(FacetedSearchView, self).extra_context() - + if self.results == []: extra['facets'] = self.form.search().facet_counts() else: extra['facets'] = self.results.facet_counts() - + return extra It updates the name of the class (generally for documentation purposes) and diff --git a/docs/who_uses.rst b/docs/who_uses.rst index ea23cdfd0..9419213dd 100644 --- a/docs/who_uses.rst +++ b/docs/who_uses.rst @@ -316,3 +316,42 @@ Using: Elasticsearch * https://gidsy.com/ * https://gidsy.com/search/ * https://gidsy.com/forum/ + + +GroundCity +---------- + +Groundcity is a Romanian dynamic real estate site. + +For real estate, forums and comments. + +Using: Whoosh + +* http://groundcity.ro/cautare/ + + +Docket Alarm +------------ + +Docket Alarm allows people to search court dockets across +the country. With it, you can search court dockets in the International Trade +Commission (ITC), the Patent Trial and Appeal Board (PTAB) and All Federal +Courts. + +Using: Elasticsearch + +* https://www.docketalarm.com/search/ITC +* https://www.docketalarm.com/search/PTAB +* https://www.docketalarm.com/search/dockets + + +Educreations +------------- + +Educreations makes it easy for anyone to teach what they know and learn +what they don't with a recordable whiteboard. Haystack is used to +provide search across users and lessons. + +Using: Solr + +* http://www.educreations.com/browse/ diff --git a/example_project/regular_app/search_indexes.py b/example_project/regular_app/search_indexes.py index 6746ef9e0..cc8398a24 100644 --- a/example_project/regular_app/search_indexes.py +++ b/example_project/regular_app/search_indexes.py @@ -19,7 +19,7 @@ class DogIndex(indexes.SearchIndex, indexes.Indexable): def get_model(self): return Dog - def index_queryset(self): + def index_queryset(self, using=None): return self.get_model().objects.filter(public=True) def prepare_toys(self, obj): diff --git a/example_project/settings.py b/example_project/settings.py index 66aab36c2..1c61214a0 100644 --- a/example_project/settings.py +++ b/example_project/settings.py @@ -27,7 +27,7 @@ }, 'xapian': { # For Xapian (requires the third-party install): - 'ENGINE': 'xapian_haystack.xapian_backend.XapianEngine', + 'ENGINE': 'xapian_backend.XapianEngine', 'PATH': os.path.join(os.path.dirname(__file__), 'xapian_index'), } } diff --git a/haystack/__init__.py b/haystack/__init__.py index 765ab5225..1a72e3996 100644 --- a/haystack/__init__.py +++ b/haystack/__init__.py @@ -1,14 +1,13 @@ import logging from django.conf import settings from django.core.exceptions import ImproperlyConfigured -from django.core import signals from haystack.constants import DEFAULT_ALIAS from haystack import signals from haystack.utils import loading __author__ = 'Daniel Lindsley' -__version__ = (2, 0, 0, 'beta') +__version__ = (2, 0, 1, 'dev') # Setup default logging. @@ -41,17 +40,17 @@ # Load the router(s). connection_router = loading.ConnectionRouter() -# Setup the signal processor. -signal_processor_path = getattr(settings, 'HAYSTACK_SIGNAL_PROCESSOR', 'haystack.signals.BaseSignalProcessor') -signal_processor_class = loading.import_class(signal_processor_path) -signal_processor = signal_processor_class(connections, connection_router) - if hasattr(settings, 'HAYSTACK_ROUTERS'): if not isinstance(settings.HAYSTACK_ROUTERS, (list, tuple)): raise ImproperlyConfigured("The HAYSTACK_ROUTERS setting must be either a list or tuple.") connection_router = loading.ConnectionRouter(settings.HAYSTACK_ROUTERS) +# Setup the signal processor. +signal_processor_path = getattr(settings, 'HAYSTACK_SIGNAL_PROCESSOR', 'haystack.signals.BaseSignalProcessor') +signal_processor_class = loading.import_class(signal_processor_path) +signal_processor = signal_processor_class(connections, connection_router) + # Per-request, reset the ghetto query log. # Probably not extraordinarily thread-safe but should only matter when @@ -62,4 +61,5 @@ def reset_search_queries(**kwargs): if settings.DEBUG: - signals.request_started.connect(reset_search_queries) + from django.core import signals as django_signals + django_signals.request_started.connect(reset_search_queries) diff --git a/haystack/backends/__init__.py b/haystack/backends/__init__.py index 606923dca..8f203b627 100644 --- a/haystack/backends/__init__.py +++ b/haystack/backends/__init__.py @@ -7,7 +7,7 @@ from django.utils import tree from django.utils.encoding import force_unicode from haystack.constants import VALID_FILTERS, FILTER_SEPARATOR, DEFAULT_ALIAS -from haystack.exceptions import MoreLikeThisError, FacetingError +from haystack.exceptions import MoreLikeThisError, FacetingError, StatsError from haystack.models import SearchResult from haystack.utils.loading import UnifiedIndex @@ -291,7 +291,7 @@ def __init__(self, using=DEFAULT_ALIAS): self.start_offset = 0 self.end_offset = None self.highlight = False - self.facets = set() + self.facets = {} self.date_facets = {} self.query_facets = [] self.narrow_queries = set() @@ -312,9 +312,10 @@ def __init__(self, using=DEFAULT_ALIAS): self._results = None self._hit_count = None self._facet_counts = None + self._stats = None self._spelling_suggestion = None self.result_class = SearchResult - + self.stats = {} from haystack import connections self._using = using self.backend = connections[self._using].get_backend() @@ -354,7 +355,7 @@ def build_params(self, spelling_query=None): kwargs['highlight'] = self.highlight if self.facets: - kwargs['facets'] = list(self.facets) + kwargs['facets'] = self.facets if self.date_facets: kwargs['date_facets'] = self.date_facets @@ -497,6 +498,17 @@ def get_facet_counts(self): return self._facet_counts + def get_stats(self): + """ + Returns the stats received from the backend. + + If the query has not been run, this will execute the query and store + the results + """ + if self._stats is None: + self.run() + return self._stats + def get_spelling_suggestion(self, preferred_query=None): """ Returns the spelling suggestion received from the backend. @@ -693,6 +705,10 @@ def more_like_this(self, model_instance): self._more_like_this = True self._mlt_instance = model_instance + def add_stats_query(self,stats_field,stats_facets): + """Adds stats and stats_facets queries for the Solr backend.""" + self.stats[stats_field] = stats_facets + def add_highlight(self): """Adds highlighting to the search results.""" self.highlight = True @@ -726,10 +742,11 @@ def add_distance(self, field, point): 'point': ensure_point(point), } - def add_field_facet(self, field): + def add_field_facet(self, field, **options): """Adds a regular facet on a field.""" from haystack import connections - self.facets.add(connections[self._using].get_unified_index().get_facet_fieldname(field)) + field_name = connections[self._using].get_unified_index().get_facet_fieldname(field) + self.facets[field_name] = options.copy() def add_date_facet(self, field, start_date, end_date, gap_by, gap_amount=1): """Adds a date-based facet on a field.""" @@ -825,6 +842,7 @@ def _clone(self, klass=None, using=None): clone.models = self.models.copy() clone.boost = self.boost.copy() clone.highlight = self.highlight + clone.stats = self.stats.copy() clone.facets = self.facets.copy() clone.date_facets = self.date_facets.copy() clone.query_facets = self.query_facets[:] @@ -837,6 +855,7 @@ def _clone(self, klass=None, using=None): clone.distance_point = self.distance_point.copy() clone._raw_query = self._raw_query clone._raw_query_params = self._raw_query_params + return clone diff --git a/haystack/backends/elasticsearch_backend.py b/haystack/backends/elasticsearch_backend.py index 9fa4f35c8..15864aefe 100644 --- a/haystack/backends/elasticsearch_backend.py +++ b/haystack/backends/elasticsearch_backend.py @@ -1,4 +1,5 @@ import datetime +import re import warnings from django.conf import settings from django.core.exceptions import ImproperlyConfigured @@ -7,7 +8,7 @@ from haystack.backends import BaseEngine, BaseSearchBackend, BaseSearchQuery, log_query from haystack.constants import ID, DJANGO_CT, DJANGO_ID, DEFAULT_OPERATOR from haystack.exceptions import MissingDependency, MoreLikeThisError -from haystack.inputs import PythonData, Clean, Exact +from haystack.inputs import PythonData, Clean, Exact, Raw from haystack.models import SearchResult from haystack.utils import get_identifier from haystack.utils import log as logging @@ -22,6 +23,11 @@ raise MissingDependency("The 'elasticsearch' backend requires the installation of 'pyelasticsearch'. Please refer to the documentation.") +DATETIME_REGEX = re.compile( + r'^(?P\d{4})-(?P\d{2})-(?P\d{2})T' + r'(?P\d{2}):(?P\d{2}):(?P\d{2})(\.\d+)?$') + + class ElasticsearchSearchBackend(BaseSearchBackend): # Word reserved by Elasticsearch for special use. RESERVED_WORDS = ( @@ -151,7 +157,7 @@ def update(self, index, iterable, commit=True): # Convert the data to make sure it's happy. for key, value in prepped_data.items(): - final_data[key] = self.conn.from_python(value) + final_data[key] = self._from_python(value) prepped_docs.append(final_data) except (requests.RequestException, pyelasticsearch.ElasticHttpError), e: @@ -161,7 +167,7 @@ def update(self, index, iterable, commit=True): # We'll log the object identifier but won't include the actual object # to avoid the possibility of that generating encoding errors while # processing the log message: - self.log.error(u"%s while preparing object for update" % e.__name__, exc_info=True, extra={ + self.log.error(u"%s while preparing object for update" % e.__class__.__name__, exc_info=True, extra={ "data": { "index": index, "object": get_identifier(obj) @@ -216,9 +222,6 @@ def clear(self, models=[], commit=True): # a ``query`` root object. :/ query = {'query_string': {'query': " OR ".join(models_to_delete)}} self.conn.delete_by_query(self.index_name, 'modelresult', query) - - if commit: - self.conn.refresh(index=self.index_name) except (requests.RequestException, pyelasticsearch.ElasticHttpError), e: if not self.silently_fail: raise @@ -309,8 +312,16 @@ def build_search_kwargs(self, query_string, sort_by=None, start_offset=0, end_of } } - if self.include_spelling is True: - warnings.warn("Elasticsearch does not handle spelling suggestions.", Warning, stacklevel=2) + if self.include_spelling: + kwargs['suggest'] = { + 'suggest': { + 'text': spelling_query or query_string, + 'term': { + # Using content_field here will result in suggestions of stemmed words. + 'field': '_all', + }, + }, + } if narrow_queries is None: narrow_queries = set() @@ -318,13 +329,21 @@ def build_search_kwargs(self, query_string, sort_by=None, start_offset=0, end_of if facets is not None: kwargs.setdefault('facets', {}) - for facet_fieldname in facets: - kwargs['facets'][facet_fieldname] = { + for facet_fieldname, extra_options in facets.items(): + facet_options = { 'terms': { 'field': facet_fieldname, 'size': 300 }, } + # Special cases for options applied at the facet level (not the terms level). + if extra_options.pop('global_scope', False): + # Renamed "global_scope" since "global" is a python keyword. + facet_options['global'] = True + if 'facet_filter' in extra_options: + facet_options['facet_filter'] = extra_options.pop('facet_filter') + facet_options['terms'].update(extra_options) + kwargs['facets'][facet_fieldname] = facet_options if date_facets is not None: kwargs.setdefault('facets', {}) @@ -346,8 +365,8 @@ def build_search_kwargs(self, query_string, sort_by=None, start_offset=0, end_of 'facet_filter': { "range": { facet_fieldname: { - 'from': self.conn.from_python(value.get('start_date')), - 'to': self.conn.from_python(value.get('end_date')), + 'from': self._from_python(value.get('start_date')), + 'to': self._from_python(value.get('end_date')), } } } @@ -564,6 +583,10 @@ def _process_results(self, raw_results, highlight=False, if result_class is None: result_class = SearchResult + if self.include_spelling and 'suggest' in raw_results: + raw_suggest = raw_results['suggest']['suggest'] + spelling_suggestion = ' '.join([word['text'] if len(word['options']) == 0 else word['options'][0]['text'] for word in raw_suggest]) + if 'facets' in raw_results: facets = { 'fields': {}, @@ -599,7 +622,7 @@ def _process_results(self, raw_results, highlight=False, if string_key in index.fields and hasattr(index.fields[string_key], 'convert'): additional_fields[string_key] = index.fields[string_key].convert(value) else: - additional_fields[string_key] = self.conn.to_python(value) + additional_fields[string_key] = self._to_python(value) del(additional_fields[DJANGO_CT]) del(additional_fields[DJANGO_ID]) @@ -682,6 +705,66 @@ def build_schema(self, fields): return (content_field_name, mapping) + def _iso_datetime(self, value): + """ + If value appears to be something datetime-like, return it in ISO format. + + Otherwise, return None. + """ + if hasattr(value, 'strftime'): + if hasattr(value, 'hour'): + return value.isoformat() + else: + return '%sT00:00:00' % value.isoformat() + + def _from_python(self, value): + """Convert more Python data types to ES-understandable JSON.""" + iso = self._iso_datetime(value) + if iso: + return iso + elif isinstance(value, str): + return unicode(value, errors='replace') # TODO: Be stricter. + elif isinstance(value, set): + return list(value) + return value + + def _to_python(self, value): + """Convert values from ElasticSearch to native Python values.""" + if isinstance(value, (int, float, complex, list, tuple, bool)): + return value + + if isinstance(value, basestring): + possible_datetime = DATETIME_REGEX.search(value) + + if possible_datetime: + date_values = possible_datetime.groupdict() + + for dk, dv in date_values.items(): + date_values[dk] = int(dv) + + return datetime.datetime( + date_values['year'], date_values['month'], + date_values['day'], date_values['hour'], + date_values['minute'], date_values['second']) + + try: + # This is slightly gross but it's hard to tell otherwise what the + # string's original type might have been. Be careful who you trust. + converted_value = eval(value) + + # Try to handle most built-in types. + if isinstance( + converted_value, + (int, list, tuple, set, dict, float, complex)): + return converted_value + except Exception: + # If it fails (SyntaxError or its ilk) or we don't trust it, + # continue on. + pass + + return value + + # Sucks that this is almost an exact copy of what's in the Solr backend, # but we can't import due to dependencies. @@ -728,7 +811,7 @@ def build_query_fragment(self, field, filter_type, value): if not isinstance(prepared_value, (set, list, tuple)): # Then convert whatever we get back to what pysolr wants if needed. - prepared_value = self.backend.conn.from_python(prepared_value) + prepared_value = self.backend._from_python(prepared_value) # 'content' is a special reserved word, much like 'pk' in # Django's ORM layer. It indicates 'no special field'. @@ -759,9 +842,9 @@ def build_query_fragment(self, field, filter_type, value): if isinstance(prepared_value, basestring): for possible_value in prepared_value.split(' '): - terms.append(filter_types[filter_type] % self.backend.conn.from_python(possible_value)) + terms.append(filter_types[filter_type] % self.backend._from_python(possible_value)) else: - terms.append(filter_types[filter_type] % self.backend.conn.from_python(prepared_value)) + terms.append(filter_types[filter_type] % self.backend._from_python(prepared_value)) if len(terms) == 1: query_frag = terms[0] @@ -771,12 +854,12 @@ def build_query_fragment(self, field, filter_type, value): in_options = [] for possible_value in prepared_value: - in_options.append(u'"%s"' % self.backend.conn.from_python(possible_value)) + in_options.append(u'"%s"' % self.backend._from_python(possible_value)) query_frag = u"(%s)" % " OR ".join(in_options) elif filter_type == 'range': - start = self.backend.conn.from_python(prepared_value[0]) - end = self.backend.conn.from_python(prepared_value[1]) + start = self.backend._from_python(prepared_value[0]) + end = self.backend._from_python(prepared_value[1]) query_frag = u'["%s" TO "%s"]' % (start, end) elif filter_type == 'exact': if value.input_type_name == 'exact': @@ -790,8 +873,9 @@ def build_query_fragment(self, field, filter_type, value): query_frag = filter_types[filter_type] % prepared_value - if len(query_frag) and not query_frag.startswith('(') and not query_frag.endswith(')'): - query_frag = "(%s)" % query_frag + if len(query_frag) and not isinstance(value, Raw): + if not query_frag.startswith('(') and not query_frag.endswith(')'): + query_frag = "(%s)" % query_frag return u"%s%s" % (index_fieldname, query_frag) @@ -842,7 +926,7 @@ def build_params(self, spelling_query=None, **kwargs): search_kwargs['end_offset'] = self.end_offset if self.facets: - search_kwargs['facets'] = list(self.facets) + search_kwargs['facets'] = self.facets if self.fields: search_kwargs['fields'] = self.fields @@ -886,6 +970,7 @@ def run_mlt(self, **kwargs): search_kwargs = { 'start_offset': self.start_offset, 'result_class': self.result_class, + 'models': self.models } if self.end_offset is not None: diff --git a/haystack/backends/simple_backend.py b/haystack/backends/simple_backend.py index 04a92c45b..7bbb21d1e 100644 --- a/haystack/backends/simple_backend.py +++ b/haystack/backends/simple_backend.py @@ -45,12 +45,16 @@ def search(self, query_string, **kwargs): hits = 0 results = [] result_class = SearchResult + models = connections[self.connection_alias].get_unified_index().get_indexed_models() if kwargs.get('result_class'): result_class = kwargs['result_class'] + if kwargs.get('models'): + models = kwargs['models'] + if query_string: - for model in connections[self.connection_alias].get_unified_index().get_indexed_models(): + for model in models: if query_string == '*': qs = model.objects.all() else: @@ -71,6 +75,7 @@ def search(self, query_string, **kwargs): hits += len(qs) for match in qs: + match.__dict__.pop('score', None) result = result_class(match._meta.app_label, match._meta.module_name, match.pk, 0, **match.__dict__) # For efficiency. result._model = match.__class__ @@ -115,7 +120,7 @@ def _build_sub_query(self, search_node): term_list.append(value.prepare(self)) - return (' ').join(map(str, term_list)) + return (' ').join(map(unicode, term_list)) class SimpleEngine(BaseEngine): diff --git a/haystack/backends/solr_backend.py b/haystack/backends/solr_backend.py index 4aa40ad3a..010b7037a 100644 --- a/haystack/backends/solr_backend.py +++ b/haystack/backends/solr_backend.py @@ -5,7 +5,7 @@ from haystack.backends import BaseEngine, BaseSearchBackend, BaseSearchQuery, log_query, EmptyResults from haystack.constants import ID, DJANGO_CT, DJANGO_ID from haystack.exceptions import MissingDependency, MoreLikeThisError -from haystack.inputs import PythonData, Clean, Exact +from haystack.inputs import PythonData, Clean, Exact, Raw from haystack.models import SearchResult from haystack.utils import get_identifier from haystack.utils import log as logging @@ -29,7 +29,7 @@ class SolrSearchBackend(BaseSearchBackend): # The '\\' must come first, so as not to overwrite the other slash replacements. RESERVED_CHARACTERS = ( '\\', '+', '-', '&&', '||', '!', '(', ')', '{', '}', - '[', ']', '^', '"', '~', '*', '?', ':', + '[', ']', '^', '"', '~', '*', '?', ':', '/', ) def __init__(self, connection_alias, **connection_options): @@ -136,7 +136,7 @@ def build_search_kwargs(self, query_string, sort_by=None, start_offset=0, end_of narrow_queries=None, spelling_query=None, within=None, dwithin=None, distance_point=None, models=None, limit_to_registered_models=None, - result_class=None): + result_class=None, stats=None): kwargs = {'fl': '* score'} if fields: @@ -183,7 +183,11 @@ def build_search_kwargs(self, query_string, sort_by=None, start_offset=0, end_of if facets is not None: kwargs['facet'] = 'on' - kwargs['facet.field'] = facets + kwargs['facet.field'] = facets.keys() + + for facet_field, options in facets.items(): + for key, value in options.items(): + kwargs['f.%s.facet.%s' % (facet_field, key)] = self.conn._from_python(value) if date_facets is not None: kwargs['facet'] = 'on' @@ -226,6 +230,15 @@ def build_search_kwargs(self, query_string, sort_by=None, start_offset=0, end_of if narrow_queries is not None: kwargs['fq'] = list(narrow_queries) + if stats: + kwargs['stats'] = "true" + + for k in stats.keys(): + kwargs['stats.field'] = k + + for facet in stats[k]: + kwargs['f.%s.stats.facet' % k] = facet + if within is not None: from haystack.utils.geo import generate_bounding_box @@ -320,11 +333,15 @@ def _process_results(self, raw_results, highlight=False, result_class=None, dist results = [] hits = raw_results.hits facets = {} + stats = {} spelling_suggestion = None if result_class is None: result_class = SearchResult + if hasattr(raw_results,'stats'): + stats = raw_results.stats.get('stats_fields',{}) + if hasattr(raw_results, 'facets'): facets = { 'fields': raw_results.facets.get('facet_fields', {}), @@ -387,6 +404,7 @@ def _process_results(self, raw_results, highlight=False, result_class=None, dist return { 'results': results, 'hits': hits, + 'stats': stats, 'facets': facets, 'spelling_suggestion': spelling_suggestion, } @@ -585,14 +603,15 @@ def build_query_fragment(self, field, filter_type, value): query_frag = filter_types[filter_type] % prepared_value - if len(query_frag) and not query_frag.startswith('(') and not query_frag.endswith(')'): - query_frag = "(%s)" % query_frag + if len(query_frag) and not isinstance(value, Raw): + if not query_frag.startswith('(') and not query_frag.endswith(')'): + query_frag = "(%s)" % query_frag return u"%s%s" % (index_fieldname, query_frag) def build_alt_parser_query(self, parser_name, query_string='', **kwargs): if query_string: - kwargs['v'] = query_string + query_string = Clean(query_string).prepare(self) kwarg_bits = [] @@ -602,13 +621,13 @@ def build_alt_parser_query(self, parser_name, query_string='', **kwargs): else: kwarg_bits.append(u"%s=%s" % (key, kwargs[key])) - return u"{!%s %s}" % (parser_name, ' '.join(kwarg_bits)) + return u'_query_:"{!%s %s}%s"' % (parser_name, Clean(' '.join(kwarg_bits)), query_string) def build_params(self, spelling_query=None, **kwargs): search_kwargs = { 'start_offset': self.start_offset, 'result_class': self.result_class - } + } order_by_list = None if self.order_by: @@ -636,7 +655,7 @@ def build_params(self, spelling_query=None, **kwargs): search_kwargs['end_offset'] = self.end_offset if self.facets: - search_kwargs['facets'] = list(self.facets) + search_kwargs['facets'] = self.facets if self.fields: search_kwargs['fields'] = self.fields @@ -659,16 +678,21 @@ def build_params(self, spelling_query=None, **kwargs): if spelling_query: search_kwargs['spelling_query'] = spelling_query + if self.stats: + search_kwargs['stats'] = self.stats + return search_kwargs - + def run(self, spelling_query=None, **kwargs): """Builds and executes the query. Returns a list of search results.""" final_query = self.build_query() search_kwargs = self.build_params(spelling_query, **kwargs) + results = self.backend.search(final_query, **search_kwargs) self._results = results.get('results', []) self._hit_count = results.get('hits', 0) self._facet_counts = self.post_process_facets(results) + self._stats = results.get('stats',{}) self._spelling_suggestion = results.get('spelling_suggestion', None) def run_mlt(self, **kwargs): @@ -680,6 +704,7 @@ def run_mlt(self, **kwargs): search_kwargs = { 'start_offset': self.start_offset, 'result_class': self.result_class, + 'models': self.models } if self.end_offset is not None: diff --git a/haystack/backends/whoosh_backend.py b/haystack/backends/whoosh_backend.py index bc9dfc00c..9990b628a 100644 --- a/haystack/backends/whoosh_backend.py +++ b/haystack/backends/whoosh_backend.py @@ -11,7 +11,7 @@ from haystack.backends import BaseEngine, BaseSearchBackend, BaseSearchQuery, log_query, EmptyResults from haystack.constants import ID, DJANGO_CT, DJANGO_ID from haystack.exceptions import MissingDependency, SearchBackendError -from haystack.inputs import PythonData, Clean, Exact +from haystack.inputs import PythonData, Clean, Exact, Raw from haystack.models import SearchResult from haystack.utils import get_identifier from haystack.utils import log as logging @@ -186,7 +186,7 @@ def update(self, index, iterable, commit=True): # We'll log the object identifier but won't include the actual object # to avoid the possibility of that generating encoding errors while # processing the log message: - self.log.error(u"%s while preparing object for update" % e.__name__, exc_info=True, extra={ + self.log.error(u"%s while preparing object for update" % e.__class__.__name__, exc_info=True, extra={ "data": { "index": index, "object": get_identifier(obj) @@ -823,7 +823,7 @@ def build_query_fragment(self, field, filter_type, value): if is_datetime is True: pv = self._convert_datetime(pv) - + if isinstance(pv, basestring) and not is_datetime: in_options.append('"%s"' % pv) else: @@ -853,8 +853,9 @@ def build_query_fragment(self, field, filter_type, value): query_frag = filter_types[filter_type] % prepared_value - if len(query_frag) and not query_frag.startswith('(') and not query_frag.endswith(')'): - query_frag = "(%s)" % query_frag + if len(query_frag) and not isinstance(value, Raw): + if not query_frag.startswith('(') and not query_frag.endswith(')'): + query_frag = "(%s)" % query_frag return u"%s%s" % (index_fieldname, query_frag) diff --git a/haystack/exceptions.py b/haystack/exceptions.py index 2bbbf4003..e70c2ca8c 100644 --- a/haystack/exceptions.py +++ b/haystack/exceptions.py @@ -29,3 +29,7 @@ class FacetingError(HaystackError): class SpatialError(HaystackError): """Raised when incorrect arguments have been provided for spatial.""" pass + +class StatsError(HaystackError): + "Raised when incorrect arguments have been provided for stats" + pass diff --git a/haystack/indexes.py b/haystack/indexes.py index 8c201765f..abfa79f83 100644 --- a/haystack/indexes.py +++ b/haystack/indexes.py @@ -6,6 +6,7 @@ from haystack import connections, connection_router from haystack.constants import ID, DJANGO_CT, DJANGO_ID, Indexable, DEFAULT_ALIAS from haystack.fields import * +from haystack.manager import SearchIndexManager from haystack.utils import get_identifier, get_facet_field_name @@ -55,6 +56,13 @@ def __new__(cls, name, bases, attrs): shadow_facet_field.set_instance_name(shadow_facet_name) attrs['fields'][shadow_facet_name] = shadow_facet_field + # Assigning default 'objects' query manager if it does not already exist + if not attrs.has_key('objects'): + try: + attrs['objects'] = SearchIndexManager(attrs['Meta'].index_label) + except (KeyError, AttributeError): + attrs['objects'] = SearchIndexManager(DEFAULT_ALIAS) + return super(DeclarativeMetaclass, cls).__new__(cls, name, bases, attrs) @@ -76,7 +84,7 @@ class NoteIndex(indexes.SearchIndex, indexes.Indexable): def get_model(self): return Note - def index_queryset(self): + def index_queryset(self, using=None): return self.get_model().objects.filter(pub_date__lte=datetime.datetime.now()) """ @@ -102,7 +110,7 @@ def get_model(self): """ raise NotImplementedError("You must provide a 'model' method for the '%r' index." % self) - def index_queryset(self): + def index_queryset(self, using=None): """ Get the default QuerySet to index when doing a full update. @@ -110,16 +118,16 @@ def index_queryset(self): """ return self.get_model()._default_manager.all() - def read_queryset(self): + def read_queryset(self, using=None): """ Get the default QuerySet for read actions. Subclasses can override this method to work with other managers. Useful when working with default managers that filter some objects. """ - return self.index_queryset() + return self.index_queryset(using=using) - def build_queryset(self, start_date=None, end_date=None): + def build_queryset(self, using=None, start_date=None, end_date=None): """ Get the default QuerySet to index when doing an index update. @@ -154,7 +162,7 @@ def build_queryset(self, start_date=None, end_date=None): warnings.warn("'SearchIndex.get_queryset' was deprecated in Haystack v2. Please rename the method 'index_queryset'.") index_qs = self.get_queryset() else: - index_qs = self.index_queryset() + index_qs = self.index_queryset(using=using) if not hasattr(index_qs, 'filter'): raise ImproperlyConfigured("The '%r' class must return a 'QuerySet' in the 'index_queryset' method." % self) @@ -269,7 +277,7 @@ def remove_object(self, instance, using=None, **kwargs): backend = self._get_backend(using) if backend is not None: - backend.remove(instance) + backend.remove(instance, **kwargs) def clear(self, using=None): """ diff --git a/haystack/management/commands/build_solr_schema.py b/haystack/management/commands/build_solr_schema.py index 741ef3683..c71fdb34d 100644 --- a/haystack/management/commands/build_solr_schema.py +++ b/haystack/management/commands/build_solr_schema.py @@ -1,7 +1,10 @@ from optparse import make_option import sys + +from django.core.exceptions import ImproperlyConfigured from django.core.management.base import BaseCommand from django.template import loader, Context +from haystack.backends.solr_backend import SolrSearchBackend from haystack.constants import ID, DJANGO_CT, DJANGO_ID, DEFAULT_OPERATOR, DEFAULT_ALIAS @@ -28,6 +31,10 @@ def handle(self, **options): def build_context(self, using): from haystack import connections, connection_router backend = connections[using].get_backend() + + if not isinstance(backend, SolrSearchBackend): + raise ImproperlyConfigured("'%s' isn't configured as a SolrEngine)." % backend.connection_alias) + content_field_name, fields = backend.build_schema(connections[using].get_unified_index().all_searchfields()) return Context({ 'content_field_name': content_field_name, diff --git a/haystack/management/commands/clear_index.py b/haystack/management/commands/clear_index.py index d99fb5bb3..179b61ed8 100644 --- a/haystack/management/commands/clear_index.py +++ b/haystack/management/commands/clear_index.py @@ -1,7 +1,7 @@ from optparse import make_option import sys + from django.core.management.base import BaseCommand -from haystack.constants import DEFAULT_ALIAS class Command(BaseCommand): @@ -10,35 +10,41 @@ class Command(BaseCommand): make_option('--noinput', action='store_false', dest='interactive', default=True, help='If provided, no prompts will be issued to the user and the data will be wiped out.' ), - make_option("-u", "--using", action="store", type="string", dest="using", default=DEFAULT_ALIAS, - help='If provided, chooses a connection to work with.' + make_option("-u", "--using", action="append", dest="using", + default=[], + help='Update only the named backend (can be used multiple times). ' + 'By default all backends will be updated.' ), ) option_list = BaseCommand.option_list + base_options - + def handle(self, **options): """Clears out the search index completely.""" from haystack import connections self.verbosity = int(options.get('verbosity', 1)) - self.using = options.get('using') - + + using = options.get('using') + if not using: + using = connections.connections_info.keys() + if options.get('interactive', True): print - print "WARNING: This will irreparably remove EVERYTHING from your search index in connection '%s'." % self.using + print "WARNING: This will irreparably remove EVERYTHING from your search index in connection '%s'." % "', '".join(using) print "Your choices after this are to restore from backups or rebuild via the `rebuild_index` command." - + yes_or_no = raw_input("Are you sure you wish to continue? [y/N] ") print - + if not yes_or_no.lower().startswith('y'): print "No action taken." sys.exit() - + if self.verbosity >= 1: print "Removing all documents from your index because you said so." - - backend = connections[self.using].get_backend() - backend.clear() - + + for backend_name in using: + backend = connections[backend_name].get_backend() + backend.clear() + if self.verbosity >= 1: print "All documents removed." diff --git a/haystack/management/commands/update_index.py b/haystack/management/commands/update_index.py index 8ffec0dc3..0c2bc837a 100755 --- a/haystack/management/commands/update_index.py +++ b/haystack/management/commands/update_index.py @@ -1,17 +1,16 @@ from datetime import timedelta from optparse import make_option +import logging import os -import warnings from django import db from django.conf import settings from django.core.exceptions import ImproperlyConfigured from django.core.management.base import LabelCommand from django.db import reset_queries -from django.utils.encoding import smart_str +from django.utils.encoding import smart_str, force_unicode from haystack import connections as haystack_connections -from haystack.constants import DEFAULT_ALIAS from haystack.query import SearchQuerySet try: @@ -70,10 +69,10 @@ def do_update(backend, index, qs, start, end, total, verbosity=1): current_qs = small_cache_qs[start:end] if verbosity >= 2: - if os.getpid() == os.getppid(): - print " indexed %s - %d of %d." % (start+1, end, total) + if hasattr(os, 'getppid') and os.getpid() == os.getppid(): + print " indexed %s - %d of %d." % (start + 1, end, total) else: - print " indexed %s - %d of %d (by %s)." % (start+1, end, total, os.getpid()) + print " indexed %s - %d of %d (by %s)." % (start + 1, end, total, os.getpid()) # FIXME: Get the right backend. backend.update(index, current_qs) @@ -121,8 +120,10 @@ class Command(LabelCommand): make_option('-r', '--remove', action='store_true', dest='remove', default=False, help='Remove objects from the index that are no longer present in the database.' ), - make_option("-u", "--using", action="store", type="string", dest="using", default=DEFAULT_ALIAS, - help='If provided, chooses a connection to work with.' + make_option("-u", "--using", action="append", dest="using", + default=[], + help='Update only the named backend (can be used multiple times). ' + 'By default all backends will be updated.' ), make_option('-k', '--workers', action='store', dest='workers', default=0, type='int', @@ -137,9 +138,11 @@ def handle(self, *items, **options): self.start_date = None self.end_date = None self.remove = options.get('remove', False) - self.using = options.get('using') self.workers = int(options.get('workers', 0)) - self.backend = haystack_connections[self.using].get_backend() + + self.backends = options.get('using') + if not self.backends: + self.backends = haystack_connections.connections_info.keys() age = options.get('age', DEFAULT_AGE) start_date = options.get('start_date') @@ -202,9 +205,18 @@ def get_models(self, label): return [get_model(app_label, model_name)] def handle_label(self, label, **options): + for using in self.backends: + try: + self.update_backend(label, using) + except: + logging.exception("Error updating %s using %s ", label, using) + raise + + def update_backend(self, label, using): from haystack.exceptions import NotHandled - unified_index = haystack_connections[self.using].get_unified_index() + backend = haystack_connections[using].get_backend() + unified_index = haystack_connections[using].get_unified_index() if self.workers > 0: import multiprocessing @@ -218,17 +230,21 @@ def handle_label(self, label, **options): continue if self.workers > 0: - # workers resetting connections leads to references to models / connections getting stale and having their connection disconnected from under them. Resetting before the loop continues and it accesses the ORM makes it better. + # workers resetting connections leads to references to models / connections getting + # stale and having their connection disconnected from under them. Resetting before + # the loop continues and it accesses the ORM makes it better. db.close_connection() - qs = index.build_queryset(start_date=self.start_date, end_date=self.end_date) + qs = index.build_queryset(using=using, start_date=self.start_date, + end_date=self.end_date) + total = qs.count() if self.verbosity >= 1: - print "Indexing %d %s." % (total, smart_str(model._meta.verbose_name_plural)) + print u"Indexing %d %s" % (total, force_unicode(model._meta.verbose_name_plural)) pks_seen = set([smart_str(pk) for pk in qs.values_list('pk', flat=True)]) - batch_size = self.batchsize or self.backend.batch_size + batch_size = self.batchsize or backend.batch_size if self.workers > 0: ghetto_queue = [] @@ -237,9 +253,9 @@ def handle_label(self, label, **options): end = min(start + batch_size, total) if self.workers == 0: - do_update(self.backend, index, qs, start, end, total, self.verbosity) + do_update(backend, index, qs, start, end, total, self.verbosity) else: - ghetto_queue.append(('do_update', model, start, end, total, self.using, self.start_date, self.end_date, self.verbosity)) + ghetto_queue.append(('do_update', model, start, end, total, using, self.start_date, self.end_date, self.verbosity)) if self.workers > 0: pool = multiprocessing.Pool(self.workers) @@ -261,9 +277,9 @@ def handle_label(self, label, **options): upper_bound = start + batch_size if self.workers == 0: - do_remove(self.backend, index, model, pks_seen, start, upper_bound) + do_remove(backend, index, model, pks_seen, start, upper_bound) else: - ghetto_queue.append(('do_remove', model, pks_seen, start, upper_bound, self.using, self.verbosity)) + ghetto_queue.append(('do_remove', model, pks_seen, start, upper_bound, using, self.verbosity)) if self.workers > 0: pool = multiprocessing.Pool(self.workers) diff --git a/haystack/manager.py b/haystack/manager.py new file mode 100644 index 000000000..c2fdf2adb --- /dev/null +++ b/haystack/manager.py @@ -0,0 +1,106 @@ +from haystack.query import SearchQuerySet, EmptySearchQuerySet + + +class SearchIndexManager(object): + def __init__(self, using=None): + super(SearchIndexManager, self).__init__() + self.using = using + + def get_search_queryset(self): + """Returns a new SearchQuerySet object. Subclasses can override this method + to easily customize the behavior of the Manager. + """ + return SearchQuerySet(using=self.using) + + def get_empty_query_set(self): + return EmptySearchQuerySet(using=self.using) + + def all(self): + return self.get_search_queryset() + + def none(self): + return self.get_empty_query_set() + + def filter(self, *args, **kwargs): + return self.get_search_queryset().filter(*args, **kwargs) + + def exclude(self, *args, **kwargs): + return self.get_search_queryset().exclude(*args, **kwargs) + + def filter_and(self, *args, **kwargs): + return self.get_search_queryset().filter_and(*args, **kwargs) + + def filter_or(self, *args, **kwargs): + return self.get_search_queryset().filter_or(*args, **kwargs) + + def order_by(self, *args): + return self.get_search_queryset().order_by(*args) + + def order_by_distance(self, **kwargs): + return self.get_search_queryset().order_by_distance(**kwargs) + + def highlight(self): + return self.get_search_queryset().highlight() + + def boost(self, term, boost): + return self.get_search_queryset().boost(term, boost) + + def facet(self, field): + return self.get_search_queryset().facet(field) + + def within(self, field, point_1, point_2): + return self.get_search_queryset().within(field, point_1, point_2) + + def dwithin(self, field, point, distance): + return self.get_search_queryset().dwithin(field, point, distance) + + def distance(self, field, point): + return self.get_search_queryset().distance(field, point) + + def date_facet(self, field, start_date, end_date, gap_by, gap_amount=1): + return self.get_search_queryset().date_facet(field, start_date, end_date, gap_by, gap_amount=1) + + def query_facet(self, field, query): + return self.get_search_queryset().query_facet(field, query) + + def narrow(self, query): + return self.get_search_queryset().narrow(query) + + def raw_search(self, query_string, **kwargs): + return self.get_search_queryset().raw_search(query_string, **kwargs) + + def load_all(self): + return self.get_search_queryset().load_all() + + def auto_query(self, query_string, fieldname='content'): + return self.get_search_queryset().auto_query(query_string, fieldname=fieldname) + + def autocomplete(self, **kwargs): + return self.get_search_queryset().autocomplete(**kwargs) + + def using(self, connection_name): + return self.get_search_queryset().using(connection_name) + + def count(self): + return self.get_search_queryset().count() + + def best_match(self): + return self.get_search_queryset().best_match() + + def latest(self, date_field): + return self.get_search_queryset().latest(date_field) + + def more_like_this(self, model_instance): + return self.get_search_queryset().more_like_this(model_instance) + + def facet_counts(self): + return self.get_search_queryset().facet_counts() + + def spelling_suggestion(self, preferred_query=None): + return self.get_search_queryset().spelling_suggestion(preferred_query=None) + + def values(self, *fields): + return self.get_search_queryset().values(*fields) + + def values_list(self, *fields, **kwargs): + return self.get_search_queryset().values_list(*fields, **kwargs) diff --git a/haystack/query.py b/haystack/query.py index 38965662f..1d44340ec 100644 --- a/haystack/query.py +++ b/haystack/query.py @@ -34,12 +34,13 @@ def __init__(self, using=None, query=None): self.log = logging.getLogger('haystack') def _determine_backend(self): + from haystack import connections # A backend has been manually selected. Use it instead. if self._using is not None: - return self._using + self.query = connections[self._using].get_query() + return # No backend, so rely on the routers to figure out what's right. - from haystack import connections hints = {} if self.query: @@ -77,7 +78,7 @@ def __setstate__(self, data_dict): def __repr__(self): data = list(self[:REPR_OUTPUT_SIZE]) - + if len(self) > REPR_OUTPUT_SIZE: data[-1] = "...(remaining elements truncated)..." @@ -205,7 +206,7 @@ def post_process_results(self, results): try: ui = connections[self.query._using].get_unified_index() index = ui.get_index(model) - objects = index.read_queryset() + objects = index.read_queryset(using=self.query._using) loaded_objects[model] = objects.in_bulk(models_pks[model]) except NotHandled: self.log.warning("Model '%s.%s' not handled by the routers.", self.app_label, self.model_name) @@ -357,10 +358,10 @@ def boost(self, term, boost): clone.query.add_boost(term, boost) return clone - def facet(self, field): + def facet(self, field, **options): """Adds faceting to a query for the provided field.""" clone = self._clone() - clone.query.add_field_facet(field) + clone.query.add_field_facet(field, **options) return clone def within(self, field, point_1, point_2): @@ -374,7 +375,23 @@ def dwithin(self, field, point, distance): clone = self._clone() clone.query.add_dwithin(field, point, distance) return clone - + + def stats(self, field): + """Adds stats to a query for the provided field.""" + return self.stats_facet(field, facet_fields=None) + + def stats_facet(self, field, facet_fields=None): + """Adds stats facet for the given field and facet_fields represents + the faceted fields.""" + clone = self._clone() + stats_facets = [] + try: + stats_facets.append(sum(facet_fields,[])) + except TypeError: + if facet_fields: stats_facets.append(facet_fields) + clone.query.add_stats_query(field,stats_facets) + return clone + def distance(self, field, point): """ Spatial: Denotes results must have distance measurements from the @@ -490,6 +507,16 @@ def facet_counts(self): clone = self._clone() return clone.query.get_facet_counts() + def stats_results(self): + """ + Returns the stats results found by the query. + """ + if self.query.has_run(): + return self.query.get_stats() + else: + clone = self._clone() + return clone.query.get_stats() + def spelling_suggestion(self, preferred_query=None): """ Returns the spelling suggestion found by the query. diff --git a/haystack/urls.py b/haystack/urls.py index 942938bcc..c699e8aa6 100644 --- a/haystack/urls.py +++ b/haystack/urls.py @@ -1,4 +1,7 @@ -from django.conf.urls.defaults import * +try: + from django.conf.urls import patterns, url +except ImportError: + from django.conf.urls.defaults import patterns, url from haystack.views import SearchView diff --git a/haystack/utils/__init__.py b/haystack/utils/__init__.py index 2e0299e88..36aa861fc 100644 --- a/haystack/utils/__init__.py +++ b/haystack/utils/__init__.py @@ -1,33 +1,77 @@ import re + from haystack.constants import ID, DJANGO_CT, DJANGO_ID from haystack.utils.highlighting import Highlighter -IDENTIFIER_REGEX = re.compile('^[\w\d_]+\.[\w\d_]+\.\d+$') +from django.conf import settings +try: + from django.utils import importlib +except ImportError: + import importlib -def get_model_ct(model): - return "%s.%s" % (model._meta.app_label, model._meta.module_name) +IDENTIFIER_REGEX = re.compile('^[\w\d_]+\.[\w\d_]+\.\d+$') -def get_identifier(obj_or_string): +def default_get_identifier(obj_or_string): """ Get an unique identifier for the object or a string representing the object. - + If not overridden, uses ... """ if isinstance(obj_or_string, basestring): if not IDENTIFIER_REGEX.match(obj_or_string): - raise AttributeError("Provided string '%s' is not a valid identifier." % obj_or_string) - + raise AttributeError(u"Provided string '%s' is not a valid identifier." % obj_or_string) + return obj_or_string - - return u"%s.%s.%s" % (obj_or_string._meta.app_label, obj_or_string._meta.module_name, obj_or_string._get_pk_val()) + + return u"%s.%s.%s" % ( + obj_or_string._meta.app_label, + obj_or_string._meta.module_name, + obj_or_string._get_pk_val() + ) + + +def _lookup_identifier_method(): + """ + If the user has set HAYSTACK_IDENTIFIER_METHOD, import it and return the method uncalled. + If HAYSTACK_IDENTIFIER_METHOD is not defined, return haystack.utils.default_get_identifier. + + This always runs at module import time. We keep the code in a function + so that it can be called from unit tests, in order to simulate the re-loading + of this module. + """ + if not hasattr(settings, 'HAYSTACK_IDENTIFIER_METHOD'): + return default_get_identifier + + module_path, method_name = settings.HAYSTACK_IDENTIFIER_METHOD.rsplit(".", 1) + + try: + module = importlib.import_module(module_path) + except ImportError: + raise ImportError(u"Unable to import module '%s' provided for HAYSTACK_IDENTIFIER_METHOD." % module_path) + + identifier_method = getattr(module, method_name, None) + + if not identifier_method: + raise AttributeError( + u"Provided method '%s' for HAYSTACK_IDENTIFIER_METHOD does not exist in '%s'." % (method_name, module_path) + ) + + return identifier_method + + +get_identifier = _lookup_identifier_method() + + +def get_model_ct(model): + return "%s.%s" % (model._meta.app_label, model._meta.module_name) def get_facet_field_name(fieldname): if fieldname in [ID, DJANGO_ID, DJANGO_CT]: return fieldname - + return "%s_exact" % fieldname diff --git a/haystack/views.py b/haystack/views.py index 6fdd6b13b..a6a33707d 100644 --- a/haystack/views.py +++ b/haystack/views.py @@ -11,7 +11,6 @@ class SearchView(object): - __name__ = 'SearchView' template = 'search/search.html' extra_context = {} query = '' @@ -151,8 +150,6 @@ def search_view(request): class FacetedSearchView(SearchView): - __name__ = 'FacetedSearchView' - def __init__(self, *args, **kwargs): # Needed to switch out the default form class. if kwargs.get('form_class') is None: diff --git a/setup.py b/setup.py index 92a90249d..799e62fdb 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,16 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from distutils.core import setup +try: + from setuptools import setup +except ImportError: + from ez_setup import use_setuptools + use_setuptools() + from setuptools import setup + setup( name='django-haystack', - version='2.0.0-beta', + version='2.0.1-dev', description='Pluggable search for Django.', author='Daniel Lindsley', author_email='daniel@toastdriven.com', @@ -34,4 +40,5 @@ 'Programming Language :: Python', 'Topic :: Utilities' ], + zip_safe=False, ) diff --git a/tests/core/custom_identifier.py b/tests/core/custom_identifier.py new file mode 100644 index 000000000..ab972081b --- /dev/null +++ b/tests/core/custom_identifier.py @@ -0,0 +1,7 @@ + +def get_identifier_method(key): + """ + Custom get_identifier method used for testing the + setting HAYSTACK_IDENTIFIER_MODULE + """ + return key diff --git a/tests/core/fixtures/bulk_data.json b/tests/core/fixtures/bulk_data.json index a4da53336..3c0178d08 100644 --- a/tests/core/fixtures/bulk_data.json +++ b/tests/core/fixtures/bulk_data.json @@ -251,5 +251,12 @@ "author": "daniel3", "pub_date": "2009-07-17 22:30:00" } + }, + { + "pk": 1, + "model": "core.ScoreMockModel", + "fields": { + "score": "42" + } } ] diff --git a/tests/core/models.py b/tests/core/models.py index 705c6661e..3bee2da59 100644 --- a/tests/core/models.py +++ b/tests/core/models.py @@ -12,10 +12,10 @@ class MockModel(models.Model): foo = models.CharField(max_length=255, blank=True) pub_date = models.DateTimeField(default=datetime.datetime.now) tag = models.ForeignKey(MockTag) - + def __unicode__(self): return self.author - + def hello(self): return 'World!' @@ -23,7 +23,7 @@ def hello(self): class AnotherMockModel(models.Model): author = models.CharField(max_length=255) pub_date = models.DateTimeField(default=datetime.datetime.now) - + def __unicode__(self): return self.author @@ -41,7 +41,7 @@ class AFourthMockModel(models.Model): author = models.CharField(max_length=255) editor = models.CharField(max_length=255) pub_date = models.DateTimeField(default=datetime.datetime.now) - + def __unicode__(self): return self.author @@ -68,3 +68,9 @@ class ASixthMockModel(models.Model): def __unicode__(self): return self.name + +class ScoreMockModel(models.Model): + score = models.CharField(max_length=10) + + def __unicode__(self): + return self.score diff --git a/tests/core/tests/__init__.py b/tests/core/tests/__init__.py index da4603983..e0056d836 100644 --- a/tests/core/tests/__init__.py +++ b/tests/core/tests/__init__.py @@ -14,3 +14,5 @@ from core.tests.templatetags import * from core.tests.views import * from core.tests.utils import * +from core.tests.management_commands import * +from core.tests.managers import * diff --git a/tests/core/tests/backends.py b/tests/core/tests/backends.py index 8b4b83e6f..6f0cba7c3 100644 --- a/tests/core/tests/backends.py +++ b/tests/core/tests/backends.py @@ -25,6 +25,16 @@ def test_load_whoosh(self): backend = loading.load_backend('haystack.backends.whoosh_backend.WhooshEngine') self.assertEqual(backend.__name__, 'WhooshEngine') + def test_load_elasticsearch(self): + try: + import pyelasticsearch + except ImportError: + warnings.warn("Pyelasticsearch doesn't appear to be installed. Unable to test loading the ElasticSearch backend.") + return + + backend = loading.load_backend('haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine') + self.assertEqual(backend.__name__, 'ElasticsearchSearchEngine') + def test_load_simple(self): backend = loading.load_backend('haystack.backends.simple_backend.SimpleEngine') self.assertEqual(backend.__name__, 'SimpleEngine') diff --git a/tests/core/tests/indexes.py b/tests/core/tests/indexes.py index 29ee880d4..8769a257b 100644 --- a/tests/core/tests/indexes.py +++ b/tests/core/tests/indexes.py @@ -67,10 +67,10 @@ def load_all_queryset(self): def get_model(self): return MockModel - def index_queryset(self): + def index_queryset(self, using=None): return MockModel.objects.all() - def read_queryset(self): + def read_queryset(self, using=None): return MockModel.objects.filter(author__in=['daniel1', 'daniel3']) def build_queryset(self, start_date=None, end_date=None): @@ -401,6 +401,19 @@ def test_remove_object(self): self.mi.remove_object(mock) self.assertEqual([(res.content_type(), res.pk) for res in self.sb.search('*')['results']], [(u'core.mockmodel', u'1'), (u'core.mockmodel', u'2'), (u'core.mockmodel', u'3')]) + + # Put it back so we can test passing kwargs. + mock = MockModel() + mock.pk = 20 + mock.author = 'daniel%s' % mock.id + mock.pub_date = datetime.datetime(2009, 1, 31, 4, 19, 0) + + self.mi.update_object(mock) + self.assertEqual(self.sb.search('*')['hits'], 4) + + self.mi.remove_object(mock, commit=False) + self.assertEqual([(res.content_type(), res.pk) for res in self.sb.search('*')['results']], [(u'core.mockmodel', u'1'), (u'core.mockmodel', u'2'), (u'core.mockmodel', u'3'), (u'core.mockmodel', u'20')]) + self.sb.clear() def test_clear(self): @@ -525,11 +538,11 @@ class GhettoAFifthMockModelSearchIndex(indexes.SearchIndex, indexes.Indexable): def get_model(self): return AFifthMockModel - def index_queryset(self): + def index_queryset(self, using=None): # Index everything, return self.get_model().objects.complete_set() - def read_queryset(self): + def read_queryset(self, using=None): return self.get_model().objects.all() @@ -539,7 +552,7 @@ class ReadQuerySetTestSearchIndex(indexes.SearchIndex, indexes.Indexable): def get_model(self): return AFifthMockModel - def read_queryset(self): + def read_queryset(self, using=None): return self.get_model().objects.complete_set() @@ -549,7 +562,7 @@ class TextReadQuerySetTestSearchIndex(indexes.SearchIndex, indexes.Indexable): def get_model(self): return AFifthMockModel - def read_queryset(self): + def read_queryset(self, using=None): return self.get_model().objects.complete_set() @@ -629,4 +642,4 @@ def test_float_integer_fields(self): self.assertTrue('average_delay' in self.yabmsi.fields) self.assertTrue(isinstance(self.yabmsi.fields['average_delay'], indexes.FloatField)) self.assertEqual(self.yabmsi.fields['average_delay'].null, False) - self.assertEqual(self.yabmsi.fields['average_delay'].index_fieldname, 'average_delay') + self.assertEqual(self.yabmsi.fields['average_delay'].index_fieldname, 'average_delay') \ No newline at end of file diff --git a/tests/core/tests/management_commands.py b/tests/core/tests/management_commands.py new file mode 100644 index 000000000..d94919b79 --- /dev/null +++ b/tests/core/tests/management_commands.py @@ -0,0 +1,57 @@ +from mock import patch, call + +from django.core.management import call_command +from django.test import TestCase + +__all__ = ['CoreManagementCommandsTestCase'] + + +class CoreManagementCommandsTestCase(TestCase): + @patch("haystack.management.commands.update_index.Command.update_backend") + def test_update_index_default_using(self, m): + """update_index uses default index when --using is not present""" + call_command('update_index') + m.assert_called_with("core", 'default') + + @patch("haystack.management.commands.update_index.Command.update_backend") + def test_update_index_using(self, m): + """update_index only applies to indexes specified with --using""" + call_command('update_index', verbosity=0, using=["eng", "fra"]) + m.assert_any_call("core", "eng") + m.assert_any_call("core", "fra") + self.assertTrue(call("core", "default") not in m.call_args_list, + "update_index should have been restricted to the index specified with --using") + + @patch("haystack.loading.ConnectionHandler.__getitem__") + def test_clear_index_default_using(self, m): + """clear_index uses default index when --using is not present""" + call_command('clear_index', verbosity=0, interactive=False) + m.assert_called_with("default") + + @patch("haystack.loading.ConnectionHandler.__getitem__") + def test_clear_index_using(self, m): + """clear_index only applies to indexes specified with --using""" + + call_command('clear_index', verbosity=0, interactive=False, using=["eng"]) + m.assert_called_with("eng") + self.assertTrue(m.return_value.get_backend.called, "backend.clear() should be called") + self.assertTrue(call("default") not in m.call_args_list, + "clear_index should have been restricted to the index specified with --using") + + @patch("haystack.loading.ConnectionHandler.__getitem__") + @patch("haystack.management.commands.update_index.Command.update_backend") + def test_rebuild_index_default_using(self, m1, m2): + """rebuild_index uses default index when --using is not present""" + + call_command('rebuild_index', verbosity=0, interactive=False) + m2.assert_called_with("default") + m1.assert_any_call("core", "default") + + @patch("haystack.loading.ConnectionHandler.__getitem__") + @patch("haystack.management.commands.update_index.Command.update_backend") + def test_rebuild_index_using(self, m1, m2): + """rebuild_index passes --using to clear_index and update_index""" + + call_command('rebuild_index', verbosity=0, interactive=False, using=["eng"]) + m2.assert_called_with("eng") + m1.assert_any_call("core", "eng") diff --git a/tests/core/tests/managers.py b/tests/core/tests/managers.py new file mode 100644 index 000000000..557d797e8 --- /dev/null +++ b/tests/core/tests/managers.py @@ -0,0 +1,188 @@ +import datetime +from django.test import TestCase +from haystack import connections +from haystack.models import SearchResult +from haystack.exceptions import FacetingError +from haystack.query import SearchQuerySet, EmptySearchQuerySet, ValuesSearchQuerySet, ValuesListSearchQuerySet +from core.models import MockModel, AnotherMockModel, CharPKMockModel, AFifthMockModel +from core.tests.views import BasicMockModelSearchIndex, BasicAnotherMockModelSearchIndex +from core.tests.mocks import CharPKMockSearchBackend +from haystack.utils.loading import UnifiedIndex +from haystack.manager import SearchIndexManager + +class CustomManager(SearchIndexManager): + def filter(self, *args, **kwargs): + return self.get_search_queryset().filter(content='foo1').filter(*args, **kwargs) + +class CustomMockModelIndexWithObjectsManager(BasicMockModelSearchIndex): + objects = CustomManager() + +class CustomMockModelIndexWithAnotherManager(BasicMockModelSearchIndex): + another = CustomManager() + +class ManagerTestCase(TestCase): + fixtures = ['bulk_data.json'] + + def setUp(self): + super(ManagerTestCase, self).setUp() + + self.search_index = BasicMockModelSearchIndex + # Update the "index". + backend = connections['default'].get_backend() + backend.clear() + backend.update(self.search_index(), MockModel.objects.all()) + + self.search_queryset = BasicMockModelSearchIndex.objects.all() + + def test_queryset(self): + self.assertTrue(isinstance(self.search_queryset, SearchQuerySet)) + + def test_none(self): + self.assertTrue(isinstance(self.search_index.objects.none(), EmptySearchQuerySet)) + + def test_filter(self): + sqs = self.search_index.objects.filter(content='foo') + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertEqual(len(sqs.query.query_filter), 1) + + def test_exclude(self): + sqs = self.search_index.objects.exclude(content='foo') + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertEqual(len(sqs.query.query_filter), 1) + + def test_filter_and(self): + sqs = self.search_index.objects.filter_and(content='foo') + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertEqual(sqs.query.query_filter.connector, 'AND') + + def test_filter_or(self): + sqs = self.search_index.objects.filter_or(content='foo') + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertEqual(sqs.query.query_filter.connector, 'OR') + + def test_order_by(self): + sqs = self.search_index.objects.order_by('foo') + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertTrue('foo' in sqs.query.order_by) + + def test_order_by_distance(self): + # Not implemented + pass + + def test_highlight(self): + sqs = self.search_index.objects.highlight() + self.assertEqual(sqs.query.highlight, True) + + def test_boost(self): + sqs = self.search_index.objects.boost('foo', 10) + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertEqual(len(sqs.query.boost.keys()), 1) + + def test_facets(self): + sqs = self.search_index.objects.facet('foo') + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertEqual(len(sqs.query.facets), 1) + + def test_within(self): + # Not implemented + pass + + def test_dwithin(self): + # Not implemented + pass + + def test_distance(self): + # Not implemented + pass + + def test_date_facets(self): + sqs = self.search_index.objects.date_facet('foo', start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by='month') + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertEqual(len(sqs.query.date_facets), 1) + + def test_query_facets(self): + sqs = self.search_index.objects.query_facet('foo', '[bar TO *]') + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertEqual(len(sqs.query.query_facets), 1) + + def test_narrow(self): + sqs = self.search_index.objects.narrow("content:foo") + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertSetEqual(set(['content:foo']), sqs.query.narrow_queries) + + def test_raw_search(self): + self.assertEqual(len(self.search_index.objects.raw_search('foo')), 23) + + def test_load_all(self): + # Models with character primary keys. + sqs = self.search_index.objects.all() + sqs.query.backend = CharPKMockSearchBackend('charpk') + results = sqs.load_all().all() + self.assertEqual(len(results._result_cache), 0) + + def test_auto_query(self): + sqs = self.search_index.objects.auto_query('test search -stuff') + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertEqual(repr(sqs.query.query_filter), '') + + # With keyword argument + sqs = self.search_index.objects.auto_query('test search -stuff', fieldname='title') + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertEqual(repr(sqs.query.query_filter), "") + + def test_autocomplete(self): + # Not implemented + pass + + def test_count(self): + self.assertEqual(SearchQuerySet().count(), 23) + self.assertEqual(self.search_index.objects.count(), 23) + + def test_best_match(self): + self.assertTrue(isinstance(self.search_index.objects.best_match(), SearchResult)) + + def test_latest(self): + self.assertTrue(isinstance(self.search_index.objects.latest('pub_date'), SearchResult)) + + def test_more_like_this(self): + mock = MockModel() + mock.id = 1 + + self.assertEqual(len(self.search_index.objects.more_like_this(mock)), 23) + + def test_facet_counts(self): + self.assertEqual(self.search_index.objects.facet_counts(), {}) + + def spelling_suggestion(self): + # Test the case where spelling support is disabled. + sqs = self.search_index.objects.filter(content='Indx') + self.assertEqual(sqs.spelling_suggestion(), None) + self.assertEqual(sqs.spelling_suggestion(preferred_query=None), None) + + def test_values(self): + sqs = self.search_index.objects.auto_query("test").values("id") + self.assert_(isinstance(sqs, ValuesSearchQuerySet)) + + def test_valueslist(self): + sqs = self.search_index.objects.auto_query("test").values_list("id") + self.assert_(isinstance(sqs, ValuesListSearchQuerySet)) + +class CustomManagerTestCase(TestCase): + fixtures = ['bulk_data.json'] + + def setUp(self): + super(CustomManagerTestCase, self).setUp() + + self.search_index_1 = CustomMockModelIndexWithObjectsManager + self.search_index_2 = CustomMockModelIndexWithAnotherManager + + def test_filter_object_manager(self): + sqs = self.search_index_1.objects.filter(content='foo') + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertEqual(len(sqs.query.query_filter), 2) + + def test_filter_another_manager(self): + sqs = self.search_index_2.another.filter(content='foo') + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertEqual(len(sqs.query.query_filter), 2) + \ No newline at end of file diff --git a/tests/core/tests/mocks.py b/tests/core/tests/mocks.py index ad63346c0..2bfd18c4d 100644 --- a/tests/core/tests/mocks.py +++ b/tests/core/tests/mocks.py @@ -46,7 +46,8 @@ def update(self, index, iterable, commit=True): def remove(self, obj, commit=True): global MOCK_INDEX_DATA - del(MOCK_INDEX_DATA[get_identifier(obj)]) + if commit == True: + del(MOCK_INDEX_DATA[get_identifier(obj)]) def clear(self, models=[], commit=True): global MOCK_INDEX_DATA diff --git a/tests/core/tests/query.py b/tests/core/tests/query.py index 3373f1881..1c32514c0 100644 --- a/tests/core/tests/query.py +++ b/tests/core/tests/query.py @@ -168,10 +168,10 @@ def test_more_like_this(self): def test_add_field_facet(self): self.bsq.add_field_facet('foo') - self.assertEqual(self.bsq.facets, set(['foo'])) + self.assertEqual(self.bsq.facets, {'foo': {}}) self.bsq.add_field_facet('bar') - self.assertEqual(self.bsq.facets, set(['foo', 'bar'])) + self.assertEqual(self.bsq.facets, {'foo': {}, 'bar': {}}) def test_add_date_facet(self): self.bsq.add_date_facet('foo', start_date=datetime.date(2009, 2, 25), end_date=datetime.date(2009, 3, 25), gap_by='day') @@ -190,6 +190,13 @@ def test_add_query_facet(self): self.bsq.add_query_facet('foo', 'baz') self.assertEqual(self.bsq.query_facets, [('foo', 'bar'), ('moof', 'baz'), ('foo', 'baz')]) + def test_add_stats(self): + self.bsq.add_stats_query('foo',['bar']) + self.assertEqual(self.bsq.stats,{'foo':['bar']}) + + self.bsq.add_stats_query('moof',['bar','baz']) + self.assertEqual(self.bsq.stats,{'foo':['bar'],'moof':['bar','baz']}) + def test_add_narrow_query(self): self.bsq.add_narrow_query('foo:bar') self.assertEqual(self.bsq.narrow_queries, set(['foo:bar'])) @@ -245,6 +252,7 @@ def test_clone(self): self.bsq.add_field_facet('foo') self.bsq.add_date_facet('foo', start_date=datetime.date(2009, 1, 1), end_date=datetime.date(2009, 1, 31), gap_by='day') self.bsq.add_query_facet('foo', 'bar') + self.bsq.add_stats_query('foo', 'bar') self.bsq.add_narrow_query('foo:bar') clone = self.bsq._clone() @@ -679,6 +687,19 @@ def test_query_facets(self): self.assertTrue(isinstance(sqs3, SearchQuerySet)) self.assertEqual(len(sqs3.query.query_facets), 3) + def test_stats(self): + sqs = self.msqs.stats_facet('foo','bar') + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertEqual(len(sqs.query.stats),1) + + sqs2 = self.msqs.stats_facet('foo','bar').stats_facet('foo','baz') + self.assertTrue(isinstance(sqs2, SearchQuerySet)) + self.assertEqual(len(sqs2.query.stats),1) + + sqs3 = self.msqs.stats_facet('foo','bar').stats_facet('moof','baz') + self.assertTrue(isinstance(sqs3, SearchQuerySet)) + self.assertEqual(len(sqs3.query.stats),2) + def test_narrow(self): sqs = self.msqs.narrow('foo:moof') self.assertTrue(isinstance(sqs, SearchQuerySet)) @@ -695,6 +716,11 @@ def test_clone(self): self.assertEqual(clone._cache_full, False) self.assertEqual(clone._using, results._using) + def test_using(self): + sqs = SearchQuerySet(using='default') + self.assertNotEqual(sqs.query, None) + self.assertEqual(sqs.query._using, 'default') + def test_chaining(self): sqs = self.msqs.filter(content='foo') self.assertTrue(isinstance(sqs, SearchQuerySet)) diff --git a/tests/core/tests/utils.py b/tests/core/tests/utils.py index d27fe2ee3..e581799a6 100644 --- a/tests/core/tests/utils.py +++ b/tests/core/tests/utils.py @@ -1,6 +1,7 @@ from django.test import TestCase +from django.test.utils import override_settings from haystack.utils import log -from haystack.utils import get_identifier, get_facet_field_name, Highlighter +from haystack.utils import get_identifier, get_facet_field_name, Highlighter, _lookup_identifier_method from core.models import MockModel @@ -16,11 +17,24 @@ def test_get_facet_field_name(self): class GetFacetFieldNameTestCase(TestCase): def test_get_identifier(self): self.assertEqual(get_identifier('core.mockmodel.1'), 'core.mockmodel.1') - + # Valid object. mock = MockModel.objects.get(pk=1) self.assertEqual(get_identifier(mock), 'core.mockmodel.1') + @override_settings(HAYSTACK_IDENTIFIER_METHOD='core.custom_identifier.get_identifier_method') + def test_haystack_identifier_method(self): + get_identifier = _lookup_identifier_method() + self.assertEqual(get_identifier('a.b.c'), 'a.b.c') + + @override_settings(HAYSTACK_IDENTIFIER_METHOD='core.custom_identifier.not_there') + def test_haystack_identifier_method_bad_path(self): + self.assertRaises(AttributeError, _lookup_identifier_method) + + @override_settings(HAYSTACK_IDENTIFIER_METHOD='core.not_there.not_there') + def test_haystack_identifier_method_bad_module(self): + self.assertRaises(ImportError, _lookup_identifier_method) + class HighlighterTestCase(TestCase): def setUp(self): @@ -28,33 +42,33 @@ def setUp(self): self.document_1 = "This is a test of the highlightable words detection. This is only a test. Were this an actual emergency, your text would have exploded in mid-air." self.document_2 = "The content of words in no particular order causes nothing to occur." self.document_3 = "%s %s" % (self.document_1, self.document_2) - + def test_find_highlightable_words(self): highlighter = Highlighter('this test') highlighter.text_block = self.document_1 self.assertEqual(highlighter.find_highlightable_words(), {'this': [0, 53, 79], 'test': [10, 68]}) - + # We don't stem for now. highlighter = Highlighter('highlight tests') highlighter.text_block = self.document_1 self.assertEqual(highlighter.find_highlightable_words(), {'highlight': [22], 'tests': []}) - + # Ignore negated bits. highlighter = Highlighter('highlight -test') highlighter.text_block = self.document_1 self.assertEqual(highlighter.find_highlightable_words(), {'highlight': [22]}) - + def test_find_window(self): # The query doesn't matter for this method, so ignore it. highlighter = Highlighter('') highlighter.text_block = self.document_1 - + # No query. self.assertEqual(highlighter.find_window({}), (0, 200)) - + # Nothing found. self.assertEqual(highlighter.find_window({'highlight': [], 'tests': []}), (0, 200)) - + # Simple cases. self.assertEqual(highlighter.find_window({'highlight': [0], 'tests': [100]}), (0, 200)) self.assertEqual(highlighter.find_window({'highlight': [99], 'tests': [199]}), (99, 299)) diff --git a/tests/elasticsearch_settings.py b/tests/elasticsearch_settings.py index 78f919d03..99cfc504f 100644 --- a/tests/elasticsearch_settings.py +++ b/tests/elasticsearch_settings.py @@ -9,6 +9,6 @@ 'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine', 'URL': 'http://127.0.0.1:9200/', 'INDEX_NAME': 'test_default', - # 'INCLUDE_SPELLING': True, + 'INCLUDE_SPELLING': True, }, } diff --git a/tests/elasticsearch_tests/tests/elasticsearch_backend.py b/tests/elasticsearch_tests/tests/elasticsearch_backend.py index e05b407eb..0842f521d 100644 --- a/tests/elasticsearch_tests/tests/elasticsearch_backend.py +++ b/tests/elasticsearch_tests/tests/elasticsearch_backend.py @@ -32,7 +32,6 @@ def clear_elasticsearch_index(): # Wipe it clean. - print 'Clearing out Elasticsearch...' raw_es = pyelasticsearch.ElasticSearch(settings.HAYSTACK_CONNECTIONS['default']['URL']) try: raw_es.delete_index(settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']) @@ -182,6 +181,10 @@ def setUp(self): connections['default']._index = self.ui self.sb = connections['default'].get_backend() + # Force the backend to rebuild the mapping each time. + self.sb.existing_mapping = {} + self.sb.setup() + self.sample_objs = [] for i in xrange(1, 4): @@ -312,7 +315,6 @@ def test_clear(self): self.sb.clear([AnotherMockModel, MockModel]) self.assertEqual(self.raw_search('*:*').get('hits', {}).get('total', 0), 0) - @unittest.expectedFailure def test_search(self): self.sb.update(self.smmi, self.sample_objs) self.assertEqual(self.raw_search('*:*')['hits']['total'], 3) @@ -327,11 +329,11 @@ def test_search(self): [[u'Indexed!\n2'], [u'Indexed!\n1'], [u'Indexed!\n3']]) self.assertEqual(self.sb.search('Indx')['hits'], 0) - self.assertEqual(self.sb.search('indax')['spelling_suggestion'], None) - self.assertEqual(self.sb.search('Indx', spelling_query='indexy')['spelling_suggestion'], None) + self.assertEqual(self.sb.search('indaxed')['spelling_suggestion'], 'indexed') + self.assertEqual(self.sb.search('arf', spelling_query='indexyd')['spelling_suggestion'], 'indexed') - self.assertEqual(self.sb.search('', facets=['name']), {'hits': 0, 'results': []}) - results = self.sb.search('Index', facets=['name']) + self.assertEqual(self.sb.search('', facets={'name': {}}), {'hits': 0, 'results': []}) + results = self.sb.search('Index', facets={'name': {}}) self.assertEqual(results['hits'], 3) self.assertEqual(results['facets']['fields']['name'], [('daniel3', 1), ('daniel2', 1), ('daniel1', 1)]) @@ -520,11 +522,6 @@ def tearDown(self): connections['default']._index = self.old_ui super(LiveElasticsearchSearchQueryTestCase, self).tearDown() - def test_get_spelling(self): - self.sq.add_filter(SQ(content='Indexy')) - self.assertEqual(self.sq.get_spelling_suggestion(), None) - self.assertEqual(self.sq.get_spelling_suggestion('indexy'), None) - def test_log_query(self): from django.conf import settings reset_search_queries() @@ -584,7 +581,6 @@ def setUp(self): global lssqstc_all_loaded if lssqstc_all_loaded is None: - print 'Reloading data...' lssqstc_all_loaded = True # Wipe it clean. @@ -1188,3 +1184,14 @@ def test_boost(self): 'core.afourthmockmodel.2', 'core.afourthmockmodel.4' ]) + + def test__to_python(self): + self.assertEqual(self.sb._to_python('abc'), 'abc') + self.assertEqual(self.sb._to_python('1'), 1) + self.assertEqual(self.sb._to_python('2653'), 2653) + self.assertEqual(self.sb._to_python('25.5'), 25.5) + self.assertEqual(self.sb._to_python('[1, 2, 3]'), [1, 2, 3]) + self.assertEqual(self.sb._to_python('{"a": 1, "b": 2, "c": 3}'), {'a': 1, 'c': 3, 'b': 2}) + self.assertEqual(self.sb._to_python('2009-05-09T16:14:00'), datetime.datetime(2009, 5, 9, 16, 14)) + self.assertEqual(self.sb._to_python('2009-05-09T00:00:00'), datetime.datetime(2009, 5, 9, 0, 0)) + self.assertEqual(self.sb._to_python(None), None) diff --git a/tests/elasticsearch_tests/tests/inputs.py b/tests/elasticsearch_tests/tests/inputs.py index 916a23bfa..0e17e2093 100644 --- a/tests/elasticsearch_tests/tests/inputs.py +++ b/tests/elasticsearch_tests/tests/inputs.py @@ -77,5 +77,5 @@ def test_altparser_init(self): def test_altparser_prepare(self): altparser = inputs.AltParser('dismax', 'douglas adams', qf='author', mm=1) - # Not supported on that backend. - self.assertEqual(altparser.prepare(self.query_obj), u"{!dismax mm=1 qf=author v='douglas adams'}") + self.assertEqual(altparser.prepare(self.query_obj), + u"""{!dismax mm=1 qf=author v='douglas adams'}""") diff --git a/tests/multipleindex/models.py b/tests/multipleindex/models.py index 3205cd5ba..2b8881a38 100644 --- a/tests/multipleindex/models.py +++ b/tests/multipleindex/models.py @@ -4,7 +4,7 @@ class Foo(models.Model): title = models.CharField(max_length=255) body = models.TextField() - + def __unicode__(self): return self.title @@ -12,6 +12,6 @@ def __unicode__(self): class Bar(models.Model): author = models.CharField(max_length=255) content = models.TextField() - + def __unicode__(self): return self.author diff --git a/tests/multipleindex/search_indexes.py b/tests/multipleindex/search_indexes.py index 7eb9c1830..487631238 100644 --- a/tests/multipleindex/search_indexes.py +++ b/tests/multipleindex/search_indexes.py @@ -11,7 +11,11 @@ def get_model(self): class FooIndex(BaseIndex, indexes.Indexable): - pass + def index_queryset(self, using=None): + qs = super(FooIndex, self).index_queryset(using=using) + if using == "filtered_whoosh": + qs = qs.filter(body__contains="1") + return qs # Import the old way & make sure things don't explode. diff --git a/tests/multipleindex/tests.py b/tests/multipleindex/tests.py index 7e48e3d84..1da782586 100644 --- a/tests/multipleindex/tests.py +++ b/tests/multipleindex/tests.py @@ -1,13 +1,16 @@ import os import shutil + from django.conf import settings from django.db import models from django.test import TestCase + from haystack import connections, connection_router from haystack.exceptions import NotHandled from haystack.query import SearchQuerySet from haystack.signals import BaseSignalProcessor, RealtimeSignalProcessor from haystack.utils.loading import UnifiedIndex + from multipleindex.search_indexes import FooIndex from multipleindex.models import Foo, Bar @@ -29,6 +32,7 @@ def setUp(self): self.bi = self.ui.get_index(Bar) self.solr_backend = connections['default'].get_backend() self.whoosh_backend = connections['whoosh'].get_backend() + self.filtered_whoosh_backend = connections['filtered_whoosh'].get_backend() foo_1 = Foo.objects.create( title='Haystack test', @@ -184,6 +188,17 @@ def test_excluded_indexes(self): # Should error, since it's not present. self.assertRaises(NotHandled, wui.get_index, Bar) + def test_filtered_index_update(self): + for i in ('whoosh', 'filtered_whoosh'): + self.fi.clear(using=i) + self.fi.update(using=i) + + results = self.whoosh_backend.search('foo') + self.assertEqual(results['hits'], 2) + + results = self.filtered_whoosh_backend.search('foo') + self.assertEqual(results['hits'], 1, "Filtered backend should only contain one record") + class TestSignalProcessor(BaseSignalProcessor): def setup(self): diff --git a/tests/multipleindex_settings.py b/tests/multipleindex_settings.py index 1a2364b6b..0ae8d23d8 100644 --- a/tests/multipleindex_settings.py +++ b/tests/multipleindex_settings.py @@ -15,6 +15,11 @@ 'PATH': mkdtemp(prefix='haystack-multipleindex-whoosh-tests-'), 'EXCLUDED_INDEXES': ['multipleindex.search_indexes.BarIndex'], }, + 'filtered_whoosh': { + 'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine', + 'PATH': mkdtemp(prefix='haystack-multipleindex-filtered-whoosh-tests-'), + 'EXCLUDED_INDEXES': ['multipleindex.search_indexes.BarIndex'], + }, } HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor' diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 000000000..31bb6aeac --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,8 @@ +Django==1.4.2 +Whoosh==2.4.1 +httplib2==0.8 +mock==1.0.1 +pyelasticsearch==0.5 +pysolr==3.0.6 +pysqlite==2.6.3 +geopy==0.95.1 diff --git a/tests/simple_tests/search_indexes.py b/tests/simple_tests/search_indexes.py index 3bebef7b5..ab348c91b 100644 --- a/tests/simple_tests/search_indexes.py +++ b/tests/simple_tests/search_indexes.py @@ -1,11 +1,18 @@ from haystack import indexes -from core.models import MockModel +from core.models import MockModel, ScoreMockModel class SimpleMockSearchIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=True) name = indexes.CharField(model_attr='author') pub_date = indexes.DateField(model_attr='pub_date') - + def get_model(self): return MockModel + +class SimpleMockScoreIndex(indexes.SearchIndex, indexes.Indexable): + text = indexes.CharField(document=True, use_template=True) + score = indexes.CharField(model_attr='score') + + def get_model(self): + return ScoreMockModel diff --git a/tests/simple_tests/tests/simple_backend.py b/tests/simple_tests/tests/simple_backend.py index 18d8b51be..54ffad796 100644 --- a/tests/simple_tests/tests/simple_backend.py +++ b/tests/simple_tests/tests/simple_backend.py @@ -1,3 +1,4 @@ +# coding: utf-8 from datetime import date from django.conf import settings from django.test import TestCase @@ -5,17 +6,9 @@ from haystack import indexes from haystack.query import SearchQuerySet from haystack.utils.loading import UnifiedIndex -from core.models import MockModel +from core.models import MockModel, ScoreMockModel from core.tests.mocks import MockSearchResult - - -class SimpleMockSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True, use_template=True) - name = indexes.CharField(model_attr='author', faceted=True) - pub_date = indexes.DateField(model_attr='pub_date') - - def get_model(self): - return MockModel +from simple_tests.search_indexes import SimpleMockSearchIndex class SimpleSearchBackendTestCase(TestCase): @@ -41,8 +34,8 @@ def test_search(self): # No query string should always yield zero results. self.assertEqual(self.backend.search(u''), {'hits': 0, 'results': []}) - self.assertEqual(self.backend.search(u'*')['hits'], 23) - self.assertEqual([result.pk for result in self.backend.search(u'*')['results']], [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]) + self.assertEqual(self.backend.search(u'*')['hits'], 24) + self.assertEqual([result.pk for result in self.backend.search(u'*')['results']], [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 1]) self.assertEqual(self.backend.search(u'daniel')['hits'], 23) self.assertEqual([result.pk for result in self.backend.search(u'daniel')['results']], [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]) @@ -79,13 +72,28 @@ def test_search(self): # Ensure that swapping the ``result_class`` works. self.assertTrue(isinstance(self.backend.search(u'index document', result_class=MockSearchResult)['results'][0], MockSearchResult)) + def test_filter_models(self): + self.backend.update(self.index, self.sample_objs) + self.assertEqual(self.backend.search(u'*', models=set([]))['hits'], 24) + self.assertEqual(self.backend.search(u'*', models=set([MockModel]))['hits'], 23) + def test_more_like_this(self): self.backend.update(self.index, self.sample_objs) - self.assertEqual(self.backend.search(u'*')['hits'], 23) + self.assertEqual(self.backend.search(u'*')['hits'], 24) # Unsupported by 'simple'. Should see empty results. self.assertEqual(self.backend.more_like_this(self.sample_objs[0])['hits'], 0) + def test_score_field_collision(self): + + index = connections['default'].get_unified_index().get_index(ScoreMockModel) + sample_objs = ScoreMockModel.objects.all() + + self.backend.update(index, self.sample_objs) + + # 42 is the in the match, which will be removed from the result + self.assertEqual(self.backend.search(u'42')['results'][0].score, 0) + class LiveSimpleSearchQuerySetTestCase(TestCase): fixtures = ['bulk_data.json'] @@ -119,6 +127,9 @@ def test_general_queries(self): self.assertTrue(len(self.sqs.exclude(name='daniel')) > 0) self.assertTrue(len(self.sqs.order_by('-pub_date')) > 0) + def test_general_queries_unicode(self): + self.assertEqual(len(self.sqs.auto_query(u'Привет')), 0) + def test_more_like_this(self): # MLT shouldn't be horribly broken. This used to throw an exception. mm1 = MockModel.objects.get(pk=1) diff --git a/tests/solr_tests/tests/inputs.py b/tests/solr_tests/tests/inputs.py index 68fdbfdba..86bbfc727 100644 --- a/tests/solr_tests/tests/inputs.py +++ b/tests/solr_tests/tests/inputs.py @@ -77,5 +77,9 @@ def test_altparser_init(self): def test_altparser_prepare(self): altparser = inputs.AltParser('dismax', 'douglas adams', qf='author', mm=1) - # Not supported on that backend. - self.assertEqual(altparser.prepare(self.query_obj), u"{!dismax mm=1 qf=author v='douglas adams'}") + self.assertEqual(altparser.prepare(self.query_obj), + u"""_query_:"{!dismax mm=1 qf=author}douglas adams\"""") + + altparser = inputs.AltParser('dismax', 'Don\'t panic', qf='text author', mm=1) + self.assertEqual(altparser.prepare(self.query_obj), + u"""_query_:"{!dismax mm=1 qf='text author'}Don't panic\"""") diff --git a/tests/solr_tests/tests/management_commands.py b/tests/solr_tests/tests/management_commands.py index 13e704139..8b8504f6e 100644 --- a/tests/solr_tests/tests/management_commands.py +++ b/tests/solr_tests/tests/management_commands.py @@ -2,6 +2,7 @@ from mock import patch import pysolr +from tempfile import mkdtemp from django import VERSION as DJANGO_VERSION from django.conf import settings @@ -144,6 +145,13 @@ def test_multiprocessing(self): call_command('update_index', verbosity=2, workers=2, batchsize=5) self.assertEqual(self.solr.search('*:*').hits, 23) + def test_build_schema_wrong_backend(self): + + settings.HAYSTACK_CONNECTIONS['whoosh'] = {'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine', + 'PATH': mkdtemp(prefix='dummy-path-'),} + + connections['whoosh']._index = self.ui + self.assertRaises(ImproperlyConfigured, call_command, 'build_solr_schema',using='whoosh', interactive=False) class AppModelManagementCommandTestCase(TestCase): fixtures = ['bulk_data.json'] diff --git a/tests/solr_tests/tests/solr_backend.py b/tests/solr_tests/tests/solr_backend.py index 1cf6296e9..a6119c544 100644 --- a/tests/solr_tests/tests/solr_backend.py +++ b/tests/solr_tests/tests/solr_backend.py @@ -12,7 +12,7 @@ from django.test import TestCase from haystack import connections, reset_search_queries from haystack import indexes -from haystack.inputs import AutoQuery +from haystack.inputs import AutoQuery, AltParser, Raw from haystack.models import SearchResult from haystack.query import SearchQuerySet, RelatedSearchQuerySet, SQ from haystack.utils.loading import UnifiedIndex @@ -162,6 +162,16 @@ def get_model(self): return ASixthMockModel +class SolrQuotingMockSearchIndex(indexes.SearchIndex, indexes.Indexable): + text = indexes.CharField(document=True, use_template=True) + + def get_model(self): + return MockModel + + def prepare_text(self, obj): + return u"""Don't panic but %s has been iñtërnâtiônàlizéð""" % obj.author + + class SolrSearchBackendTestCase(TestCase): def setUp(self): super(SolrSearchBackendTestCase, self).setUp() @@ -178,6 +188,7 @@ def setUp(self): self.ui.build(indexes=[self.smmi]) connections['default']._index = self.ui self.sb = connections['default'].get_backend() + self.sq = connections['default'].get_query() self.sample_objs = [] @@ -319,8 +330,8 @@ def test_search(self): self.assertEqual(self.sb.search('indax')['spelling_suggestion'], 'index') self.assertEqual(self.sb.search('Indx', spelling_query='indexy')['spelling_suggestion'], 'index') - self.assertEqual(self.sb.search('', facets=['name']), {'hits': 0, 'results': []}) - results = self.sb.search('Index', facets=['name']) + self.assertEqual(self.sb.search('', facets={'name': {}}), {'hits': 0, 'results': []}) + results = self.sb.search('Index', facets={'name': {}}) self.assertEqual(results['hits'], 3) self.assertEqual(results['facets']['fields']['name'], [('daniel1', 1), ('daniel2', 1), ('daniel3', 1)]) @@ -335,6 +346,11 @@ def test_search(self): self.assertEqual(results['hits'], 3) self.assertEqual(results['facets']['queries'], {'name:[* TO e]': 3}) + self.assertEqual(self.sb.search('', stats={}), {'hits':0,'results':[]}) + results = self.sb.search('*:*', stats={'name':['name']}) + self.assertEqual(results['hits'], 3) + self.assertEqual(results['stats']['name']['count'], 3) + self.assertEqual(self.sb.search('', narrow_queries=set(['name:daniel1'])), {'hits': 0, 'results': []}) results = self.sb.search('Index', narrow_queries=set(['name:daniel1'])) self.assertEqual(results['hits'], 1) @@ -358,6 +374,55 @@ def test_search(self): # Restore. settings.HAYSTACK_LIMIT_TO_REGISTERED_MODELS = old_limit_to_registered_models + def test_altparser_query(self): + self.sb.update(self.smmi, self.sample_objs) + + results = self.sb.search(AltParser('dismax', "daniel1", qf='name', mm=1).prepare(self.sq)) + self.assertEqual(results['hits'], 1) + + # This should produce exactly the same result since all we have are mockmodel instances but we simply + # want to confirm that using the AltParser doesn't break other options: + results = self.sb.search(AltParser('dismax', 'daniel1', qf='name', mm=1).prepare(self.sq), + narrow_queries=set(('django_ct:core.mockmodel', ))) + self.assertEqual(results['hits'], 1) + + results = self.sb.search(AltParser('dismax', '+indexed +daniel1', qf='text name', mm=1).prepare(self.sq)) + self.assertEqual(results['hits'], 1) + + self.sq.add_filter(SQ(name=AltParser('dismax', 'daniel1', qf='name', mm=1))) + self.sq.add_filter(SQ(text='indexed')) + + new_q = self.sq._clone() + new_q._reset() + + new_q.add_filter(SQ(name='daniel1')) + new_q.add_filter(SQ(text=AltParser('dismax', 'indexed', qf='text'))) + + results = new_q.get_results() + self.assertEqual(len(results), 1) + self.assertEqual(results[0].id, 'core.mockmodel.1') + + def test_raw_query(self): + self.sb.update(self.smmi, self.sample_objs) + + # Ensure that the raw bits have proper parenthesis. + new_q = self.sq._clone() + new_q._reset() + new_q.add_filter(SQ(content=Raw("{!dismax qf='title^2 text' mm=1}my query"))) + + results = new_q.get_results() + self.assertEqual(len(results), 0) + + def test_altparser_quoting(self): + test_objs = [ + MockModel(id=1, author="Foo d'Bar", pub_date=datetime.date.today()), + MockModel(id=2, author="Baaz Quuz", pub_date=datetime.date.today()), + ] + self.sb.update(SolrQuotingMockSearchIndex(), test_objs) + + results = self.sb.search(AltParser('dismax', "+don't +quuz", qf='text').prepare(self.sq)) + self.assertEqual(results['hits'], 1) + def test_more_like_this(self): self.sb.update(self.smmi, self.sample_objs) self.assertEqual(self.raw_solr.search('*:*').hits, 3) @@ -797,6 +862,11 @@ def test_auto_query(self): self.assertEqual(sqs.query.build_query(), u'("pants\\:rule")') self.assertEqual(len(sqs), 0) + sqs = self.sqs.auto_query('Canon+PowerShot+ELPH+(Black)') + self.assertEqual(sqs.query.build_query(), u'Canon\\+PowerShot\\+ELPH\\+\\(Black\\)') + sqs = sqs.filter(tags__in=['cameras', 'electronics']) + self.assertEqual(len(sqs), 0) + # Regressions def test_regression_proper_start_offsets(self): diff --git a/tests/solr_tests/tests/solr_query.py b/tests/solr_tests/tests/solr_query.py index 2cb196cc6..784acda0d 100644 --- a/tests/solr_tests/tests/solr_query.py +++ b/tests/solr_tests/tests/solr_query.py @@ -1,7 +1,7 @@ import datetime from django.test import TestCase from haystack import connections -from haystack.inputs import Exact +from haystack.inputs import Exact, AltParser from haystack.models import SearchResult from haystack.query import SQ from core.models import MockModel, AnotherMockModel @@ -74,6 +74,16 @@ def test_build_query_multiple_filter_types(self): self.sq.add_filter(SQ(rating__range=[3, 5])) self.assertEqual(self.sq.build_query(), u'((why) AND pub_date:([* TO "2009-02-10 01:59:00"]) AND author:({"daniel" TO *}) AND created:({* TO "2009-02-12 12:13:00"}) AND title:(["B" TO *]) AND id:("1" OR "2" OR "3") AND rating:(["3" TO "5"]))') + def test_build_complex_altparser_query(self): + self.sq.add_filter(SQ(content=AltParser('dismax', "Don't panic", qf='text'))) + self.sq.add_filter(SQ(pub_date__lte=Exact('2009-02-10 01:59:00'))) + self.sq.add_filter(SQ(author__gt='daniel')) + self.sq.add_filter(SQ(created__lt=Exact('2009-02-12 12:13:00'))) + self.sq.add_filter(SQ(title__gte='B')) + self.sq.add_filter(SQ(id__in=[1, 2, 3])) + self.sq.add_filter(SQ(rating__range=[3, 5])) + self.assertEqual(self.sq.build_query(), u'((_query_:"{!dismax qf=text}Don\'t panic") AND pub_date:([* TO "2009-02-10 01:59:00"]) AND author:({"daniel" TO *}) AND created:({* TO "2009-02-12 12:13:00"}) AND title:(["B" TO *]) AND id:("1" OR "2" OR "3") AND rating:(["3" TO "5"]))') + def test_build_query_multiple_filter_types_with_datetimes(self): self.sq.add_filter(SQ(content='why')) self.sq.add_filter(SQ(pub_date__lte=datetime.datetime(2009, 2, 10, 1, 59, 0))) @@ -107,7 +117,7 @@ def test_build_query_wildcard_filter_types(self): def test_clean(self): self.assertEqual(self.sq.clean('hello world'), 'hello world') self.assertEqual(self.sq.clean('hello AND world'), 'hello and world') - self.assertEqual(self.sq.clean('hello AND OR NOT TO + - && || ! ( ) { } [ ] ^ " ~ * ? : \ world'), 'hello and or not to \\+ \\- \\&& \\|| \\! \\( \\) \\{ \\} \\[ \\] \\^ \\" \\~ \\* \\? \\: \\\\ world') + self.assertEqual(self.sq.clean('hello AND OR NOT TO + - && || ! ( ) { } [ ] ^ " ~ * ? : \ / world'), 'hello and or not to \\+ \\- \\&& \\|| \\! \\( \\) \\{ \\} \\[ \\] \\^ \\" \\~ \\* \\? \\: \\\\ \\/ world') self.assertEqual(self.sq.clean('so please NOTe i am in a bAND and bORed'), 'so please NOTe i am in a bAND and bORed') def test_build_query_with_models(self): diff --git a/tox.ini b/tox.ini index 2a499700b..bcc3c21c7 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,7 @@ deps = pysolr poster whoosh - pyelasticsearch + pyelasticsearch>=0.4 httplib2 python-dateutil geopy