Skip to content

Commit

Permalink
Make cql2_like_to_es() understand escaped backslashes (#286)
Browse files Browse the repository at this point in the history
**Related Issue(s):**

- Closes #285

**Description:**

This is a break/fix PR. The first commit adds a suite of tests that
document correct LIKE-to-wildcard query value conversions, and then
fixes the `cql2_like_to_es()` code to correctly process escaped
backslashes.

**PR Checklist:**

- [x] Code is formatted and linted (run `pre-commit run --all-files`)
- [x] Tests pass (run `make test`)
_(`make test` fails because the Docker container port ranges don't
match, but CI, which doesn't use `make test`, passes)_
- [ ] Documentation has been updated to reflect changes, if applicable
      _n/a, no docs in repo_
- [x] Changes are added to the changelog
  • Loading branch information
kurtmckee authored Aug 23, 2024
1 parent 9a51574 commit ace0c7a
Show file tree
Hide file tree
Showing 3 changed files with 76 additions and 15 deletions.
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).

## [Unreleased]

### Changed
- Support escaped backslashes in CQL2 `LIKE` queries, and reject invalid (or incomplete) escape sequences. [#286](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/286)

## [v3.0.0] - 2024-08-14

### Changed
Expand Down
40 changes: 25 additions & 15 deletions stac_fastapi/core/stac_fastapi/core/extensions/filter.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,31 +17,41 @@
from enum import Enum
from typing import Any, Dict

_cql2_like_patterns = re.compile(r"\\.|[%_]|\\$")
_valid_like_substitutions = {
"\\\\": "\\",
"\\%": "%",
"\\_": "_",
"%": "*",
"_": "?",
}


def _replace_like_patterns(match: re.Match) -> str:
pattern = match.group()
try:
return _valid_like_substitutions[pattern]
except KeyError:
raise ValueError(f"'{pattern}' is not a valid escape sequence")


def cql2_like_to_es(string: str) -> str:
"""
Convert CQL2 wildcard characters to Elasticsearch wildcard characters. Specifically, it converts '_' to '?' and '%' to '*', handling escape characters properly.
Convert CQL2 "LIKE" characters to Elasticsearch "wildcard" characters.
Args:
string (str): The string containing CQL2 wildcard characters.
Returns:
str: The converted string with Elasticsearch compatible wildcards.
Raises:
ValueError: If an invalid escape sequence is encountered.
"""
# Translate '%' and '_' only if they are not preceded by a backslash '\'
percent_pattern = r"(?<!\\)%"
underscore_pattern = r"(?<!\\)_"
# Remove the escape character before '%' or '_'
escape_pattern = r"\\(?=[_%])"

# Replace '%' with '*' for broad wildcard matching
string = re.sub(percent_pattern, "*", string)
# Replace '_' with '?' for single character wildcard matching
string = re.sub(underscore_pattern, "?", string)
# Remove the escape character used in the CQL2 format
string = re.sub(escape_pattern, "", string)

return string
return _cql2_like_patterns.sub(
repl=_replace_like_patterns,
string=string,
)


class LogicalOp(str, Enum):
Expand Down
46 changes: 46 additions & 0 deletions stac_fastapi/tests/extensions/test_cql2_like_to_es.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
import pytest

from stac_fastapi.core.extensions.filter import cql2_like_to_es


@pytest.mark.parametrize(
"cql2_value, expected_es_value",
(
# no-op
("", ""),
# backslash
("\\\\", "\\"),
# percent
("%", "*"),
(r"\%", "%"),
(r"\\%", r"\*"),
(r"\\\%", r"\%"),
# underscore
("_", "?"),
(r"\_", "_"),
(r"\\_", r"\?"),
(r"\\\_", r"\_"),
),
)
def test_cql2_like_to_es_success(cql2_value: str, expected_es_value: str) -> None:
"""Verify CQL2 LIKE query strings are converted correctly."""

assert cql2_like_to_es(cql2_value) == expected_es_value


@pytest.mark.parametrize(
"cql2_value",
(
pytest.param("\\", id="trailing backslash escape"),
pytest.param("\\1", id="invalid escape sequence"),
),
)
def test_cql2_like_to_es_invalid(cql2_value: str) -> None:
"""Verify that incomplete or invalid escape sequences are rejected.
CQL2 currently doesn't appear to define how to handle invalid escape sequences.
This test assumes that undefined behavior is caught.
"""

with pytest.raises(ValueError):
cql2_like_to_es(cql2_value)

0 comments on commit ace0c7a

Please sign in to comment.