diff --git a/src/AElf.EntityMapping.Elasticsearch/IElasticsearchQueryableFactory.cs b/src/AElf.EntityMapping.Elasticsearch/IElasticsearchQueryableFactory.cs index 73955e8..fa16054 100644 --- a/src/AElf.EntityMapping.Elasticsearch/IElasticsearchQueryableFactory.cs +++ b/src/AElf.EntityMapping.Elasticsearch/IElasticsearchQueryableFactory.cs @@ -1,4 +1,6 @@ using AElf.EntityMapping.Elasticsearch.Linq; +using AElf.EntityMapping.Elasticsearch.Options; +using Microsoft.Extensions.Options; using Nest; using Volo.Abp.Domain.Entities; @@ -14,14 +16,18 @@ public class ElasticsearchQueryableFactory : IElasticsearchQueryableFac where TEntity : class, IEntity { private readonly ICollectionNameProvider _collectionNameProvider; + private readonly ElasticsearchOptions _elasticsearchOptions; - public ElasticsearchQueryableFactory(ICollectionNameProvider collectionNameProvider) + public ElasticsearchQueryableFactory(ICollectionNameProvider collectionNameProvider, + IOptions elasticsearchOptions) { _collectionNameProvider = collectionNameProvider; + _elasticsearchOptions = elasticsearchOptions.Value; } - public ElasticsearchQueryable Create(IElasticClient client, string index = null) + public ElasticsearchQueryable Create(IElasticClient client, + string index = null) { - return new ElasticsearchQueryable(client, _collectionNameProvider, index); + return new ElasticsearchQueryable(client, _collectionNameProvider, index, _elasticsearchOptions); } } \ No newline at end of file diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs index b856206..77ecbe7 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs @@ -1,6 +1,9 @@ using System.Collections.ObjectModel; using System.Linq.Expressions; +using AElf.EntityMapping.Elasticsearch.Options; using AElf.EntityMapping.Linq; +using AElf.EntityMapping.Options; +using Microsoft.Extensions.Options; using Nest; using Remotion.Linq; using Remotion.Linq.Clauses; @@ -13,12 +16,15 @@ public class ElasticsearchGeneratorQueryModelVisitor : QueryModelVisitorBase { private readonly PropertyNameInferrerParser _propertyNameInferrerParser; private readonly INodeVisitor _nodeVisitor; + private readonly ElasticsearchOptions _elasticsearchOptions; private QueryAggregator QueryAggregator { get; set; } = new QueryAggregator(); - public ElasticsearchGeneratorQueryModelVisitor(PropertyNameInferrerParser propertyNameInferrerParser) + public ElasticsearchGeneratorQueryModelVisitor(PropertyNameInferrerParser propertyNameInferrerParser, + ElasticsearchOptions elasticsearchOptions) { _propertyNameInferrerParser = propertyNameInferrerParser; _nodeVisitor = new NodeVisitor(); + _elasticsearchOptions = elasticsearchOptions; } public QueryAggregator GenerateElasticQuery(QueryModel queryModel) @@ -48,7 +54,7 @@ public override void VisitMainFromClause(MainFromClause fromClause, QueryModel q public override void VisitWhereClause(WhereClause whereClause, QueryModel queryModel, int index) { - var tree = new GeneratorExpressionTreeVisitor(_propertyNameInferrerParser); + var tree = new GeneratorExpressionTreeVisitor(_propertyNameInferrerParser, _elasticsearchOptions); tree.Visit(whereClause.Predicate); if (QueryAggregator.Query == null) { diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs index efd629c..1d5b521 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs @@ -3,6 +3,7 @@ using System.Dynamic; using System.Linq.Expressions; using AElf.EntityMapping.Elasticsearch.Exceptions; +using AElf.EntityMapping.Elasticsearch.Options; using Elasticsearch.Net; using Nest; using Newtonsoft.Json; @@ -23,16 +24,20 @@ public class ElasticsearchQueryExecutor: IQueryExecutor private readonly JsonSerializerSettings _deserializerSettings; private readonly ICollectionNameProvider _collectionNameProvider; private const int ElasticQueryLimit = 10000; + private readonly ElasticsearchOptions _elasticsearchOptions; public ElasticsearchQueryExecutor(IElasticClient elasticClient, - ICollectionNameProvider collectionNameProvider, string index) + ICollectionNameProvider collectionNameProvider, string index, + ElasticsearchOptions elasticsearchOptions) { _elasticClient = elasticClient; _collectionNameProvider = collectionNameProvider; _index = index; _propertyNameInferrerParser = new PropertyNameInferrerParser(_elasticClient); + _elasticsearchOptions = elasticsearchOptions; _elasticsearchGeneratorQueryModelVisitor = - new ElasticsearchGeneratorQueryModelVisitor(_propertyNameInferrerParser); + new ElasticsearchGeneratorQueryModelVisitor(_propertyNameInferrerParser, + _elasticsearchOptions); _deserializerSettings = new JsonSerializerSettings { // Nest maps TimeSpan as a long (TimeSpan ticks) diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryable.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryable.cs index 074c5bd..6e0cf25 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryable.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryable.cs @@ -1,4 +1,5 @@ using System.Linq.Expressions; +using AElf.EntityMapping.Elasticsearch.Options; using AElf.EntityMapping.Linq; using Nest; using Remotion.Linq; @@ -10,10 +11,10 @@ public class ElasticsearchQueryable : QueryableBase, IElasticsearchQueryab where T : class, IEntity { public ElasticsearchQueryable(IElasticClient elasticClient, ICollectionNameProvider collectionNameProvider, - string index) - : base(new DefaultQueryProvider(typeof(ElasticsearchQueryable<>), + string index, ElasticsearchOptions elasticsearchOptions) + : base(new DefaultQueryProvider(typeof(ElasticsearchQueryable<>), QueryParserFactory.Create(), - new ElasticsearchQueryExecutor(elasticClient, collectionNameProvider, index))) + new ElasticsearchQueryExecutor(elasticClient, collectionNameProvider, index, elasticsearchOptions))) { } diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs index 271373d..34bae13 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs @@ -1,5 +1,5 @@ using System.Linq.Expressions; -using System.Reflection; +using AElf.EntityMapping.Elasticsearch.Options; using Elasticsearch.Net; using Remotion.Linq.Clauses; using Remotion.Linq.Clauses.Expressions; @@ -11,6 +11,7 @@ namespace AElf.EntityMapping.Elasticsearch.Linq public class GeneratorExpressionTreeVisitor : ThrowingExpressionVisitor { private readonly PropertyNameInferrerParser _propertyNameInferrerParser; + private readonly ElasticsearchOptions _elasticsearchOptions; private object Value { get; set; } private string PropertyName { get; set; } @@ -20,9 +21,11 @@ public class GeneratorExpressionTreeVisitor : ThrowingExpressionVisitor public IDictionary QueryMap { get; } = new Dictionary(); - public GeneratorExpressionTreeVisitor(PropertyNameInferrerParser propertyNameInferrerParser) + public GeneratorExpressionTreeVisitor(PropertyNameInferrerParser propertyNameInferrerParser, + ElasticsearchOptions elasticsearchOptions) { _propertyNameInferrerParser = propertyNameInferrerParser; + _elasticsearchOptions = elasticsearchOptions; } protected override Expression VisitUnary(UnaryExpression expression) @@ -203,7 +206,14 @@ protected override Expression VisitSubQuery(SubQueryExpression expression) case ContainsResultOperator containsResultOperator: Visit(containsResultOperator.Item); Visit(expression.QueryModel.MainFromClause.FromExpression); - + + //Check if the number of items in the Terms query array within the Contains clause is too large. + if (expression.QueryModel.MainFromClause + .FromExpression is ConstantExpression constantExpression) + { + CheckTermsArrayLength(constantExpression); + } + // Handling different types query = GetDifferentTypesTermsQueryNode(); @@ -290,6 +300,13 @@ private void HandleNestedContains(SubQueryExpression subQueryExpression, Express if (subQueryExpression == null || expression == null) throw new ArgumentNullException("SubQueryExpression or expression cannot be null."); + //Check if the number of items in the Terms query array within the Contains clause is too large. + if (subQueryExpression.QueryModel.MainFromClause + .FromExpression is ConstantExpression constantExpression) + { + CheckTermsArrayLength(constantExpression); + } + foreach (var resultOperator in subQueryExpression.QueryModel.ResultOperators) { switch (resultOperator) @@ -375,18 +392,16 @@ private Node GetDifferentTypesTermsQueryNode() return query; } - private string GetMemberName(Expression expression) + private void CheckTermsArrayLength(ConstantExpression constantExpression) { - if (expression is MemberExpression memberExpression) - return memberExpression.Member.Name; - throw new InvalidOperationException("Expression does not represent a member access."); - } - - private object GetValueFromExpression(Expression expression) - { - if (expression is ConstantExpression constantExpression) - return constantExpression.Value; - throw new InvalidOperationException("Expression is not a constant."); + if (constantExpression.Value is System.Collections.IEnumerable objectList) + { + var count = objectList.Cast().Count(); + if (count > _elasticsearchOptions.TermsArrayMaxLength) + { + throw new Exception($"The array input for Terms query is too large, exceeding {_elasticsearchOptions.TermsArrayMaxLength} items."); + } + } } protected override Expression VisitQuerySourceReference(QuerySourceReferenceExpression expression) diff --git a/src/AElf.EntityMapping.Elasticsearch/Options/ElasticsearchOptions.cs b/src/AElf.EntityMapping.Elasticsearch/Options/ElasticsearchOptions.cs index bf6e534..464d72b 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Options/ElasticsearchOptions.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Options/ElasticsearchOptions.cs @@ -9,4 +9,5 @@ public class ElasticsearchOptions public int NumberOfReplicas { get; set; } = 1; public Refresh Refresh { get; set; } = Refresh.False; public int MaxResultWindow { get; set; } = 10000; + public int TermsArrayMaxLength { get; set; } = 100; } \ No newline at end of file diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs index a0ca41f..5808035 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs @@ -619,11 +619,11 @@ public async Task GetList_Terms_Test() var queryable = await _elasticsearchRepository.GetQueryableAsync(); - var predicates = inputs - .Select(s => (Expression>)(info => info.BlockHash == s)) - .Aggregate((prev, next) => prev.Or(next)); - var filterList_predicate = queryable.Where(predicates).ToList(); - filterList_predicate.Count.ShouldBe(3); + // var predicates = inputs + // .Select(s => (Expression>)(info => info.BlockHash == s)) + // .Aggregate((prev, next) => prev.Or(next)); + // var filterList_predicate = queryable.Where(predicates).ToList(); + // filterList_predicate.Count.ShouldBe(3); var filterList = queryable.Where(item => inputs.Contains(item.BlockHash)).ToList(); filterList.Count.ShouldBe(3); @@ -659,12 +659,12 @@ public async Task GetNestedList_Terms_Test() 101, 103 }; - var queryable_predicate = await _transactionIndexRepository.GetQueryableAsync(); - var predicates = inputs - .Select(s => (Expression>)(info => info.LogEvents.Any(x => x.BlockHeight == s))) - .Aggregate((prev, next) => prev.Or(next)); - var filterList_predicate = queryable_predicate.Where(predicates).ToList(); - filterList_predicate.Count.ShouldBe(2); + // var queryable_predicate = await _transactionIndexRepository.GetQueryableAsync(); + // var predicates = inputs + // .Select(s => (Expression>)(info => info.LogEvents.Any(x => x.BlockHeight == s))) + // .Aggregate((prev, next) => prev.Or(next)); + // var filterList_predicate = queryable_predicate.Where(predicates).ToList(); + // filterList_predicate.Count.ShouldBe(2); Expression> mustQuery = item => item.LogEvents.Any(x => inputs.Contains(x.BlockHeight));