Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Release v1.3.0 #38

Merged
merged 2 commits into from
Oct 22, 2024
Merged

Release v1.3.0 #38

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
afe3b7b
Feature: Support querying with Terms syntax (#36)
AElfBourneShi Oct 22, 2024
7d4e68f
Merge branch 'master' into release/1.3.0
jason-aelf Oct 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,21 @@
{
public static string GetErrorMessage(IResponse response)
{
return response.ServerError == null ? "Unknown error." : response.ServerError.ToString();
if (response.ServerError == null)
{

Check warning on line 10 in src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs

View check run for this annotation

Codecov / codecov/patch

src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs#L10 

Added line #L10 was not covered by tests
if (response.OriginalException == null)
{
return "Unknown error.";

Check warning on line 13 in src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs

View check run for this annotation

Codecov / codecov/patch

src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs#L12-L13 

Added lines #L12 - L13 were not covered by tests
}

if (response.OriginalException.InnerException == null)
{
return response.OriginalException.Message;

Check warning on line 18 in src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs

View check run for this annotation

Codecov / codecov/patch

src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs#L17-L18 

Added lines #L17 - L18 were not covered by tests
}

return response.OriginalException.InnerException.Message;

Check warning on line 21 in src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs

View check run for this annotation

Codecov / codecov/patch

src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs#L21 

Added line #L21 was not covered by tests
}

return response.ServerError.ToString();
}
}
2 changes: 1 addition & 1 deletion src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ public ElasticsearchClientProvider(IOptions<ElasticsearchOptions> options)
var uris = options.Value.Uris.ConvertAll(x => new Uri(x));
var connectionPool = new StaticConnectionPool(uris);
var settings = new ConnectionSettings(connectionPool);
// .DisableDirectStreaming();
// .DisableDirectStreaming()
// .OnRequestCompleted(callDetails =>
// {
// // Print Request DSL
Expand Down
12 changes: 9 additions & 3 deletions src/AElf.EntityMapping.Elasticsearch/IElasticsearchQueryableFactory.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
using AElf.EntityMapping.Elasticsearch.Linq;
using AElf.EntityMapping.Elasticsearch.Options;
using Microsoft.Extensions.Options;
using Nest;
using Volo.Abp.Domain.Entities;

Expand All @@ -14,14 +16,18 @@ public class ElasticsearchQueryableFactory<TEntity> : IElasticsearchQueryableFac
where TEntity : class, IEntity
{
private readonly ICollectionNameProvider<TEntity> _collectionNameProvider;
private readonly ElasticsearchOptions _elasticsearchOptions;

public ElasticsearchQueryableFactory(ICollectionNameProvider<TEntity> collectionNameProvider)
public ElasticsearchQueryableFactory(ICollectionNameProvider<TEntity> collectionNameProvider,
IOptions<ElasticsearchOptions> elasticsearchOptions)
{
_collectionNameProvider = collectionNameProvider;
_elasticsearchOptions = elasticsearchOptions.Value;
}

public ElasticsearchQueryable<TEntity> Create(IElasticClient client, string index = null)
public ElasticsearchQueryable<TEntity> Create(IElasticClient client,
string index = null)
{
return new ElasticsearchQueryable<TEntity>(client, _collectionNameProvider, index);
return new ElasticsearchQueryable<TEntity>(client, _collectionNameProvider, index, _elasticsearchOptions);
}
}
10 changes: 8 additions & 2 deletions src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
using System.Collections.ObjectModel;
using System.Linq.Expressions;
using AElf.EntityMapping.Elasticsearch.Options;
using AElf.EntityMapping.Linq;
using AElf.EntityMapping.Options;
using Microsoft.Extensions.Options;
using Nest;
using Remotion.Linq;
using Remotion.Linq.Clauses;
Expand All @@ -13,12 +16,15 @@ public class ElasticsearchGeneratorQueryModelVisitor<TU> : QueryModelVisitorBase
{
private readonly PropertyNameInferrerParser _propertyNameInferrerParser;
private readonly INodeVisitor _nodeVisitor;
private readonly ElasticsearchOptions _elasticsearchOptions;
private QueryAggregator QueryAggregator { get; set; } = new QueryAggregator();

public ElasticsearchGeneratorQueryModelVisitor(PropertyNameInferrerParser propertyNameInferrerParser)
public ElasticsearchGeneratorQueryModelVisitor(PropertyNameInferrerParser propertyNameInferrerParser,
ElasticsearchOptions elasticsearchOptions)
{
_propertyNameInferrerParser = propertyNameInferrerParser;
_nodeVisitor = new NodeVisitor();
_elasticsearchOptions = elasticsearchOptions;
}

public QueryAggregator GenerateElasticQuery<T>(QueryModel queryModel)
Expand Down Expand Up @@ -48,7 +54,7 @@ public override void VisitMainFromClause(MainFromClause fromClause, QueryModel q

public override void VisitWhereClause(WhereClause whereClause, QueryModel queryModel, int index)
{
var tree = new GeneratorExpressionTreeVisitor<TU>(_propertyNameInferrerParser);
var tree = new GeneratorExpressionTreeVisitor<TU>(_propertyNameInferrerParser, _elasticsearchOptions);
tree.Visit(whereClause.Predicate);
if (QueryAggregator.Query == null)
{
Expand Down
9 changes: 7 additions & 2 deletions src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using System.Dynamic;
using System.Linq.Expressions;
using AElf.EntityMapping.Elasticsearch.Exceptions;
using AElf.EntityMapping.Elasticsearch.Options;
using Elasticsearch.Net;
using Nest;
using Newtonsoft.Json;
Expand All @@ -23,16 +24,20 @@ public class ElasticsearchQueryExecutor<TEntity>: IQueryExecutor
private readonly JsonSerializerSettings _deserializerSettings;
private readonly ICollectionNameProvider<TEntity> _collectionNameProvider;
private const int ElasticQueryLimit = 10000;
private readonly ElasticsearchOptions _elasticsearchOptions;

public ElasticsearchQueryExecutor(IElasticClient elasticClient,
ICollectionNameProvider<TEntity> collectionNameProvider, string index)
ICollectionNameProvider<TEntity> collectionNameProvider, string index,
ElasticsearchOptions elasticsearchOptions)
{
_elasticClient = elasticClient;
_collectionNameProvider = collectionNameProvider;
_index = index;
_propertyNameInferrerParser = new PropertyNameInferrerParser(_elasticClient);
_elasticsearchOptions = elasticsearchOptions;
_elasticsearchGeneratorQueryModelVisitor =
new ElasticsearchGeneratorQueryModelVisitor<TEntity>(_propertyNameInferrerParser);
new ElasticsearchGeneratorQueryModelVisitor<TEntity>(_propertyNameInferrerParser,
_elasticsearchOptions);
_deserializerSettings = new JsonSerializerSettings
{
// Nest maps TimeSpan as a long (TimeSpan ticks)
Expand Down
7 changes: 4 additions & 3 deletions src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryable.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System.Linq.Expressions;
using AElf.EntityMapping.Elasticsearch.Options;
using AElf.EntityMapping.Linq;
using Nest;
using Remotion.Linq;
Expand All @@ -10,10 +11,10 @@ public class ElasticsearchQueryable<T> : QueryableBase<T>, IElasticsearchQueryab
where T : class, IEntity
{
public ElasticsearchQueryable(IElasticClient elasticClient, ICollectionNameProvider<T> collectionNameProvider,
string index)
: base(new DefaultQueryProvider(typeof(ElasticsearchQueryable<>),
string index, ElasticsearchOptions elasticsearchOptions)
: base(new DefaultQueryProvider(typeof(ElasticsearchQueryable<>),
QueryParserFactory.Create(),
new ElasticsearchQueryExecutor<T>(elasticClient, collectionNameProvider, index)))
new ElasticsearchQueryExecutor<T>(elasticClient, collectionNameProvider, index, elasticsearchOptions)))
{
}

Expand Down
179 changes: 143 additions & 36 deletions src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System.Linq.Expressions;
using AElf.EntityMapping.Elasticsearch.Options;
using Elasticsearch.Net;
using Remotion.Linq.Clauses;
using Remotion.Linq.Clauses.Expressions;
Expand All @@ -10,6 +11,7 @@ namespace AElf.EntityMapping.Elasticsearch.Linq
public class GeneratorExpressionTreeVisitor<T> : ThrowingExpressionVisitor
{
private readonly PropertyNameInferrerParser _propertyNameInferrerParser;
private readonly ElasticsearchOptions _elasticsearchOptions;

private object Value { get; set; }
private string PropertyName { get; set; }
Expand All @@ -19,9 +21,11 @@ public class GeneratorExpressionTreeVisitor<T> : ThrowingExpressionVisitor
public IDictionary<Expression, Node> QueryMap { get; } =
new Dictionary<Expression, Node>();

public GeneratorExpressionTreeVisitor(PropertyNameInferrerParser propertyNameInferrerParser)
public GeneratorExpressionTreeVisitor(PropertyNameInferrerParser propertyNameInferrerParser,
ElasticsearchOptions elasticsearchOptions)
{
_propertyNameInferrerParser = propertyNameInferrerParser;
_elasticsearchOptions = elasticsearchOptions;
}

protected override Expression VisitUnary(UnaryExpression expression)
Expand Down Expand Up @@ -129,22 +133,22 @@ protected override Expression VisitMethodCall(MethodCallExpression expression)

// private string GetFullNameKey(MemberExpression memberExpression)
// {
// var key = _propertyNameInferrerParser.Parser(memberExpression.Member.Name);
// while (memberExpression.Expression != null)
// {
// memberExpression = memberExpression.Expression as MemberExpression;
// if (memberExpression == null)
// {
// break;
// }
//
// key = _propertyNameInferrerParser.Parser(memberExpression.Member.Name + "." + key);
// return key;
// }
//
// return key;
// var key = _propertyNameInferrerParser.Parser(memberExpression.Member.Name);
// while (memberExpression.Expression != null)
// {
// memberExpression = memberExpression.Expression as MemberExpression;
// if (memberExpression == null)
// {
// break;
// }
//
// key = _propertyNameInferrerParser.Parser(memberExpression.Member.Name + "." + key);
// return key;
// }

//
// return key;
// }

private string GetFullPropertyPath(Expression expression)
{
switch (expression)
Expand All @@ -162,6 +166,7 @@ private string GetFullPropertyPath(Expression expression)
var collectionPath = GetFullPropertyPath(methodCallExpression.Object);
return collectionPath; // Returns the path of the collection directly, without adding an index
}

break;
}

Expand Down Expand Up @@ -203,15 +208,15 @@ protected override Expression VisitSubQuery(SubQueryExpression expression)
Visit(containsResultOperator.Item);
Visit(expression.QueryModel.MainFromClause.FromExpression);

if (containsResultOperator.Item.Type == typeof(Guid))
//Check if the number of items in the Terms query array within the Contains clause is too large.
if (expression.QueryModel.MainFromClause
.FromExpression is ConstantExpression constantExpression)
{
query = new TermsNode(PropertyName, ((IEnumerable<Guid>)Value).Select(x => x.ToString()));
CheckTermsArrayLength(constantExpression);
}

if (containsResultOperator.Item.Type == typeof(Guid?))
{
query = new TermsNode(PropertyName, ((IEnumerable<Guid?>)Value).Select(x => x.ToString()));
}
// Handling different types
query = GetDifferentTypesTermsQueryNode();

QueryMap[expression] = ParseQuery(query);
break;
Expand All @@ -227,23 +232,32 @@ protected override Expression VisitSubQuery(SubQueryExpression expression)

foreach (var whereClause in whereClauses)
{
Visit(whereClause.Predicate);
Node tmp = (Node)QueryMap[whereClause.Predicate].Clone();
QueryMap[expression] = tmp;
QueryMap[expression].IsSubQuery = true;
QueryMap[expression].SubQueryPath = from; //from.ToLower();
QueryMap[expression].SubQueryFullPath = fullPath;
// VisitBinarySetSubQuery((BinaryExpression)whereClause.Predicate, from, fullPath, true);
BinaryExpression predicate = (BinaryExpression)whereClause.Predicate;
if (predicate.Left is BinaryExpression)
if (whereClause.Predicate is SubQueryExpression subQueryExpression)
{
VisitBinarySetSubQuery((BinaryExpression)predicate.Left, from, fullPath, true);
HandleNestedContains(subQueryExpression, expression, from,
fullPath);
}

if (predicate.Right is BinaryExpression)
else
{
VisitBinarySetSubQuery((BinaryExpression)predicate.Right, from, fullPath, true);
Visit(whereClause.Predicate);
Node tmp = (Node)QueryMap[whereClause.Predicate].Clone();
QueryMap[expression] = tmp;
QueryMap[expression].IsSubQuery = true;
QueryMap[expression].SubQueryPath = from; //from.ToLower();
QueryMap[expression].SubQueryFullPath = fullPath;
// VisitBinarySetSubQuery((BinaryExpression)whereClause.Predicate, from, fullPath, true);
BinaryExpression predicate = (BinaryExpression)whereClause.Predicate;
if (predicate.Left is BinaryExpression)
{
VisitBinarySetSubQuery((BinaryExpression)predicate.Left, from, fullPath, true);
}

if (predicate.Right is BinaryExpression)
{
VisitBinarySetSubQuery((BinaryExpression)predicate.Right, from, fullPath, true);
}
}

}

break;
Expand Down Expand Up @@ -281,6 +295,98 @@ protected override Expression VisitSubQuery(SubQueryExpression expression)
return expression;
}

private void HandleNestedContains(SubQueryExpression subQueryExpression, Expression expression,
string subQueryPath, string subQueryFullPath)
{
if (subQueryExpression == null || expression == null)
throw new ArgumentNullException("SubQueryExpression or expression cannot be null.");

//Check if the number of items in the Terms query array within the Contains clause is too large.
if (subQueryExpression.QueryModel.MainFromClause
.FromExpression is ConstantExpression constantExpression)
{
CheckTermsArrayLength(constantExpression);
}

foreach (var resultOperator in subQueryExpression.QueryModel.ResultOperators)
{
switch (resultOperator)
{
case ContainsResultOperator containsResultOperator:
Visit(containsResultOperator.Item);
Visit(subQueryExpression.QueryModel.MainFromClause.FromExpression);
break;
}
}

Node query;
query = GetDifferentTypesTermsQueryNode();

QueryMap[expression] = ParseQuery(query);
QueryMap[expression].IsSubQuery = true;
QueryMap[expression].SubQueryPath = subQueryPath; //from.ToLower();
QueryMap[expression].SubQueryFullPath = subQueryFullPath;
}

private Node GetDifferentTypesTermsQueryNode()
{
Node query;
if (PropertyType == typeof(Guid))
{
query = GetTermsNode<Guid>();
}
else if (PropertyType == typeof(int))
{
query = GetTermsNode<int>();
}
else if (PropertyType == typeof(long))
{
query = GetTermsNode<long>();
}
else if (PropertyType == typeof(double))
{
query = GetTermsNode<double>();
}
else if (PropertyType == typeof(bool))
{
query = GetTermsNode<bool>();
}
else if (PropertyType == typeof(DateTime))
{
query = new TermsNode(PropertyName,
((IEnumerable<DateTime>)Value).Select(x => x.ToString("o")));
}
else if (PropertyType == typeof(string))
{
query = new TermsNode(PropertyName, (IEnumerable<string>)Value);
}
else
{
throw new NotSupportedException($"Type {PropertyType.Name} is not supported for Terms queries.");
}

return query;
}

private TermsNode GetTermsNode<T>()
{
return new TermsNode(PropertyName,
((IEnumerable<T>)Value).Select(x => x.ToString()));
}

private void CheckTermsArrayLength(ConstantExpression constantExpression)
{
if (constantExpression.Value is System.Collections.IEnumerable objectList)
{
var count = objectList.Cast<object>().Count();
if (count > _elasticsearchOptions.TermsArrayMaxLength)
{
throw new ArgumentException(
$"The array input for Terms query is too large, exceeding {_elasticsearchOptions.TermsArrayMaxLength} items.");
}
}
}

protected override Expression VisitQuerySourceReference(QuerySourceReferenceExpression expression)
{
return expression;
Expand Down Expand Up @@ -597,7 +703,8 @@ private object ConvertEnumValue(Type entityType, string propertyName, object val
return (int)enumValue;
}

protected void VisitBinarySetSubQuery(BinaryExpression expression, string path, string fullPath, bool parentIsSubQuery)
protected void VisitBinarySetSubQuery(BinaryExpression expression, string path, string fullPath,
bool parentIsSubQuery)
{
if (expression.Left is BinaryExpression && expression.Right is ConstantExpression)
{
Expand Down
Loading