Add full text search (#713)

* add search

* Update search

* Update

* add postgres_search and update docs

* Fix CI

* Fix CI

* Fix test

CHANGELOG.rst

@@ -18,6 +18,7 @@ Changelog
 - Filter backward FK fields with `IS NULL` and `NOT IS NULL` filters (#700)
 - Add `select_for_update` in `update_or_create`. (#702)
 - Add `Model.select_for_update`.
+- Add `__search` full text search to queryset.
 
 0.17.1
 ------

docs/contrib/mysql.rst

@@ -18,3 +18,10 @@ Fields
 MySQL specific fields.
 
 .. autoclass:: tortoise.contrib.mysql.fields.GeometryField
+
+Search
+======
+
+MySQL full text search.
+
+.. autoclass:: tortoise.contrib.mysql.search.SearchCriterion

docs/contrib/postgres.rst

@@ -22,3 +22,16 @@ Fields
 Postgres specific fields.
 
 .. autoclass:: tortoise.contrib.postgres.fields.TSVectorField
+
+Functions
+=========
+
+.. autoclass:: tortoise.contrib.postgres.functions.ToTsVector
+.. autoclass:: tortoise.contrib.postgres.functions.ToTsQuery
+
+Search
+======
+
+Postgres full text search.
+
+.. autoclass:: tortoise.contrib.postgres.search.SearchCriterion

docs/query.rst

@@ -265,6 +265,7 @@ When using ``.filter()`` method you can use number of modifiers to field names t
 - ``endswith`` - if field ends with value
 - ``iendswith`` - case insensitive ``endswith``
 - ``iexact`` - case insensitive equals
+- ``search`` - full text search
 
 Specially, you can filter date part with one of following, note that current only support PostgreSQL and MySQL, but not sqlite:
 

tests/test_queryset.py

@@ -418,3 +418,10 @@ async def test_select_related_with_two_same_models(self):
         self.assertEqual(tree.parent.name, parent_node.name)
         self.assertEqual(tree.child.pk, child_node.pk)
         self.assertEqual(tree.child.name, child_node.name)
+
+    @test.requireCapability(dialect="postgres")
+    async def test_postgres_search(self):
+        name = "hello world"
+        await Tournament.create(name=name)
+        ret = await Tournament.filter(name__search="hello").first()
+        self.assertEqual(ret.name, name)

tortoise/backends/asyncpg/executor.py

@@ -3,14 +3,22 @@
 
 import asyncpg
 from pypika import Parameter
+from pypika.terms import Term
 
 from tortoise import Model
 from tortoise.backends.base.executor import BaseExecutor
+from tortoise.contrib.postgres.search import SearchCriterion
+from tortoise.filters import search
+
+
+def postgres_search(field: Term, value: Term):
+    return SearchCriterion(field, expr=value)
 
 
 class AsyncpgExecutor(BaseExecutor):
     EXPLAIN_PREFIX = "EXPLAIN (FORMAT JSON, VERBOSE)"
     DB_NATIVE = BaseExecutor.DB_NATIVE | {bool, uuid.UUID}
+    FILTER_FUNC_OVERRIDE = {search: postgres_search}
 
     def parameter(self, pos: int) -> Parameter:
         return Parameter("$%d" % (pos + 1,))

tortoise/backends/mysql/executor.py

@@ -4,6 +4,7 @@
 
 from tortoise import Model
 from tortoise.backends.base.executor import BaseExecutor
+from tortoise.contrib.mysql.search import SearchCriterion
 from tortoise.fields import BigIntField, IntField, SmallIntField
 from tortoise.filters import (
     Like,
@@ -16,6 +17,7 @@
     insensitive_ends_with,
     insensitive_exact,
     insensitive_starts_with,
+    search,
     starts_with,
 )
 
@@ -81,6 +83,10 @@ def mysql_insensitive_ends_with(field: Term, value: str) -> Criterion:
     )
 
 
+def mysql_search(field: Term, value: str):
+    return SearchCriterion(field, expr=StrWrapper(value))
+
+
 class MySQLExecutor(BaseExecutor):
     FILTER_FUNC_OVERRIDE = {
         contains: mysql_contains,
@@ -90,6 +96,7 @@ class MySQLExecutor(BaseExecutor):
         insensitive_contains: mysql_insensitive_contains,
         insensitive_starts_with: mysql_insensitive_starts_with,
         insensitive_ends_with: mysql_insensitive_ends_with,
+        search: mysql_search,
     }
     EXPLAIN_PREFIX = "EXPLAIN FORMAT=JSON"
 

tortoise/contrib/mysql/search.py

@@ -0,0 +1,43 @@
+from enum import Enum
+from typing import Any, Optional
+
+from pypika.enums import Comparator
+from pypika.terms import BasicCriterion
+from pypika.terms import Function as PypikaFunction
+from pypika.terms import Term
+
+
+class Comp(Comparator):  # type: ignore
+    search = " "
+
+
+class Mode(Enum):
+    NATURAL_LANGUAGE_MODE = "IN NATURAL LANGUAGE MODE"
+    NATURAL_LANGUAGE_MODE_WITH_QUERY_EXPRESSION = "IN NATURAL LANGUAGE MODE WITH QUERY EXPANSION"
+    BOOL_MODE = "IN BOOLEAN MODE"
+    WITH_QUERY_EXPRESSION = "WITH QUERY EXPANSION"
+
+
+class Match(PypikaFunction):  # type: ignore
+    def __init__(self, *columns: Term):
+        super(Match, self).__init__("MATCH", *columns)
+
+
+class Against(PypikaFunction):  # type: ignore
+    def __init__(self, expr: Term, mode: Optional[Mode] = None):
+        super(Against, self).__init__("AGAINST", expr)
+        self.mode = mode
+
+    def get_special_params_sql(self, **kwargs: Any) -> Any:
+        if not self.mode:
+            return ""
+        return self.mode.value
+
+
+class SearchCriterion(BasicCriterion):  # type: ignore
+    """
+    Only support for CharField, TextField with full search indexes.
+    """
+
+    def __init__(self, *columns: Term, expr: Term, mode: Optional[Mode] = None):
+        super().__init__(Comp.search, Match(*columns), Against(expr, mode))

tortoise/contrib/postgres/functions.py

@@ -0,0 +1,19 @@
+from pypika.terms import Function, Term
+
+
+class ToTsVector(Function):  # type: ignore
+    """
+    to to_tsvector function
+    """
+
+    def __init__(self, field: Term):
+        super(ToTsVector, self).__init__("TO_TSVECTOR", field)
+
+
+class ToTsQuery(Function):  # type: ignore
+    """
+    to_tsquery function
+    """
+
+    def __init__(self, field: Term):
+        super(ToTsQuery, self).__init__("TO_TSQUERY", field)

tortoise/contrib/postgres/search.py

@@ -0,0 +1,13 @@
+from pypika.enums import Comparator
+from pypika.terms import BasicCriterion, Term
+
+from tortoise.contrib.postgres.functions import ToTsQuery, ToTsVector
+
+
+class Comp(Comparator):  # type: ignore
+    search = " @@ "
+
+
+class SearchCriterion(BasicCriterion):  # type: ignore
+    def __init__(self, field: Term, expr: Term):
+        super().__init__(Comp.search, ToTsVector(field), ToTsQuery(expr))

tortoise/filters.py

@@ -153,6 +153,11 @@ def contains(field: Term, value: str) -> Criterion:
     return Like(Cast(field, SqlTypes.VARCHAR), field.wrap_constant(f"%{escape_like(value)}%"))
 
 
+def search(field: Term, value: str):
+    # will be override in each executor
+    pass
+
+
 def starts_with(field: Term, value: str) -> Criterion:
     return Like(Cast(field, SqlTypes.VARCHAR), field.wrap_constant(f"{escape_like(value)}%"))
 
@@ -387,6 +392,12 @@ def get_filters_for_field(
             "operator": starts_with,
             "value_encoder": string_encoder,
         },
+        f"{field_name}__search": {
+            "field": actual_field_name,
+            "source_field": source_field,
+            "operator": search,
+            "value_encoder": string_encoder,
+        },
         f"{field_name}__endswith": {
             "field": actual_field_name,
             "source_field": source_field,