Skip to content

Commit

Permalink
feat: support terms query
Browse files Browse the repository at this point in the history
  • Loading branch information
@AElfBourneShi
AElfBourneShi committed Oct 16, 2024
1 parent eb3add5 commit 35f6bc6
Show file tree
Hide file tree
Showing 4 changed files with 189 additions and 21 deletions.
17 changes: 16 additions & 1 deletion src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,21 @@ public class ElasticsearchResponseHelper
{
public static string GetErrorMessage(IResponse response)
{
return response.ServerError == null ? "Unknown error." : response.ServerError.ToString();
if (response.ServerError == null)
{
if (response.OriginalException == null)
{
return "Unknown error.";
}

if (response.OriginalException.InnerException == null)
{
return response.OriginalException.Message;
}

return response.OriginalException.InnerException.Message;
}

return response.ServerError.ToString();
}
}
30 changes: 15 additions & 15 deletions src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,21 +20,21 @@ public ElasticsearchClientProvider(IOptions<ElasticsearchOptions> options)
{
var uris = options.Value.Uris.ConvertAll(x => new Uri(x));
var connectionPool = new StaticConnectionPool(uris);
var settings = new ConnectionSettings(connectionPool);
// .DisableDirectStreaming();
// .OnRequestCompleted(callDetails =>
// {
// // Print Request DSL
// if (callDetails.RequestBodyInBytes != null)
// {
// Console.WriteLine($"Request JSON: {Encoding.UTF8.GetString(callDetails.RequestBodyInBytes)}");
// }
// // // Print Response Data
// // if (callDetails.ResponseBodyInBytes != null)
// // {
// // Console.WriteLine($"Response JSON: {Encoding.UTF8.GetString(callDetails.ResponseBodyInBytes)}");
// // }
// });
var settings = new ConnectionSettings(connectionPool)
.DisableDirectStreaming()
.OnRequestCompleted(callDetails =>
{
// Print Request DSL
if (callDetails.RequestBodyInBytes != null)
{
Console.WriteLine($"Request JSON: {Encoding.UTF8.GetString(callDetails.RequestBodyInBytes)}");
}
// // Print Response Data
// if (callDetails.ResponseBodyInBytes != null)
// {
// Console.WriteLine($"Response JSON: {Encoding.UTF8.GetString(callDetails.ResponseBodyInBytes)}");
// }
});
_elasticClient = new ElasticClient(settings);
}

Expand Down
84 changes: 79 additions & 5 deletions src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System.Linq.Expressions;
using System.Reflection;
using Elasticsearch.Net;
using Remotion.Linq.Clauses;
using Remotion.Linq.Clauses.Expressions;
Expand Down Expand Up @@ -202,15 +203,41 @@ protected override Expression VisitSubQuery(SubQueryExpression expression)
case ContainsResultOperator containsResultOperator:
Visit(containsResultOperator.Item);
Visit(expression.QueryModel.MainFromClause.FromExpression);

if (containsResultOperator.Item.Type == typeof(Guid))

// PropertyName = _propertyNameInferrerParser.Parser(GetFullPropertyPath(expression.QueryModel.MainFromClause.FromExpression));
Type itemType = containsResultOperator.Item.Type;
Type nonNullableType = Nullable.GetUnderlyingType(itemType) ?? itemType;

// Handling different types
if (itemType == typeof(Guid) || nonNullableType == typeof(Guid))
{
query = new TermsNode(PropertyName, ((IEnumerable<Guid>)Value).Select(x => x.ToString()));
}

if (containsResultOperator.Item.Type == typeof(Guid?))
else if (itemType == typeof(Guid?) || nonNullableType == typeof(Guid?))
{
query = new TermsNode(PropertyName, ((IEnumerable<Guid?>)Value).Select(x => x?.ToString()));
}
else if (nonNullableType == typeof(int) || nonNullableType == typeof(int?)) {
query = new TermsNode(PropertyName, ((IEnumerable<int>)Value).Select(x => x.ToString()));
}
else if (nonNullableType == typeof(long) || nonNullableType == typeof(long?)) {
query = new TermsNode(PropertyName, ((IEnumerable<long>)Value).Select(x => x.ToString()));
}
else if (nonNullableType == typeof(double) || nonNullableType == typeof(double?)) {
query = new TermsNode(PropertyName, ((IEnumerable<double>)Value).Select(x => x.ToString()));
}
else if (nonNullableType == typeof(DateTime) || nonNullableType == typeof(DateTime?)) {
query = new TermsNode(PropertyName, ((IEnumerable<DateTime>)Value).Select(x => x.ToString("o"))); // ISO 8601 format
}
else if (nonNullableType == typeof(bool) || nonNullableType == typeof(bool?)) {
query = new TermsNode(PropertyName, ((IEnumerable<bool>)Value).Select(x => x.ToString()));
}
else if (nonNullableType == typeof(string)) {
query = new TermsNode(PropertyName, (IEnumerable<string>)Value);
}
else
{
query = new TermsNode(PropertyName, ((IEnumerable<Guid?>)Value).Select(x => x.ToString()));
throw new NotSupportedException($"Type {nonNullableType.Name} is not supported for Terms queries.");
}

QueryMap[expression] = ParseQuery(query);
Expand All @@ -234,6 +261,11 @@ protected override Expression VisitSubQuery(SubQueryExpression expression)
QueryMap[expression].SubQueryPath = from; //from.ToLower();
QueryMap[expression].SubQueryFullPath = fullPath;
// VisitBinarySetSubQuery((BinaryExpression)whereClause.Predicate, from, fullPath, true);
if (whereClause.Predicate is SubQueryExpression subQueryExpression)
{
// HandleNestedContains(subQueryExpression, tmp);
return base.VisitSubQuery(expression);
}
BinaryExpression predicate = (BinaryExpression)whereClause.Predicate;
if (predicate.Left is BinaryExpression)
{
Expand Down Expand Up @@ -281,6 +313,48 @@ protected override Expression VisitSubQuery(SubQueryExpression expression)
return expression;
}

// private void HandleNestedContains(SubQueryExpression subQueryExpression, Node parent)
// {
// if (subQueryExpression == null || parent == null)
// throw new ArgumentNullException("SubQueryExpression or parent Node cannot be null.");
//
// PropertyName = _propertyNameInferrerParser.Parser(GetFullPropertyPath(subQueryExpression));
// PropertyType = Nullable.GetUnderlyingType(subQueryExpression.Type) ?? subQueryExpression.Type;
// Node query;
// if (PropertyType == typeof(Guid) || PropertyType == typeof(Guid))
// {
// query = new TermsNode(PropertyName, ((IEnumerable<Guid>)Value).Select(x => x.ToString()));
// }
// else if (PropertyType == typeof(int) || PropertyType == typeof(int))
// {
// query = new TermsNode(PropertyName, ((IEnumerable<int>)Value).Select(x => x.ToString()));
// }
// else if (PropertyType == typeof(string))
// {
// query = new TermsNode(PropertyName, ((IEnumerable<bool>)Value).Select(x => x.ToString()));
// }
// else
// {
// throw new NotSupportedException($"Type {PropertyType.Name} is not supported for Contains queries.");
// }
// // parent.(query);
// QueryMap[subQueryExpression] = ParseQuery(query);
// }

private string GetMemberName(Expression expression)
{
if (expression is MemberExpression memberExpression)
return memberExpression.Member.Name;
throw new InvalidOperationException("Expression does not represent a member access.");
}

private object GetValueFromExpression(Expression expression)
{
if (expression is ConstantExpression constantExpression)
return constantExpression.Value;
throw new InvalidOperationException("Expression is not a constant.");
}

protected override Expression VisitQuerySourceReference(QuerySourceReferenceExpression expression)
{
return expression;
Expand Down
79 changes: 79 additions & 0 deletions test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -593,6 +593,85 @@ public async Task GetList_Nested_Test()
filterList.Count.ShouldBe(1);
}

[Fact]
public async Task GetList_Terms_Test()
{
for (int i = 1; i <= 7; i++)
{
var blockIndex = new BlockIndex
{
Id = "block" + i,
BlockHash = "BlockHash" + i,
BlockHeight = i,
BlockTime = DateTime.Now.AddDays(-10 + i),
LogEventCount = i,
ChainId = "AELF"
};
await _elasticsearchRepository.AddAsync(blockIndex);
}

List<string> inputs = new List<string>()
{
"BlockHash2",
"BlockHash3",
"BlockHash4"
};

var queryable = await _elasticsearchRepository.GetQueryableAsync();

var predicates = inputs
.Select(s => (Expression<Func<BlockIndex, bool>>)(info => info.BlockHash == s))
.Aggregate((prev, next) => prev.Or(next));
var filterList_predicate = queryable.Where(predicates).ToList();

var filterList = queryable.Where(item => inputs.Contains(item.BlockHash)).ToList();
filterList.Count.ShouldBe(3);

List<long> heights = new List<long>()
{
4, 5
};
Expression<Func<BlockIndex, bool>> mustQuery = item => heights.Contains(item.BlockHeight);
var filterList_heights = queryable.Where(mustQuery).ToList();
filterList_heights.Count.ShouldBe(2);
}

[Fact]
public async Task GetNestedList_Terms_Test()
{
//clear data for unit test
ClearTransactionIndex("AELF", 100, 110);

Thread.Sleep(2000);
//Unit Test 14
var transaction_100 = MockNewTransactionEtoData(100, false, "token_contract_address", "DonateResourceToken");
var transaction_101 = MockNewTransactionEtoData(101, false, "", "");
var transaction_103 = MockNewTransactionEtoData(103, false, "consensus_contract_address", "UpdateValue");
var transaction_110 = MockNewTransactionEtoData(110, true, "consensus_contract_address", "UpdateTinyBlockInformation");
await _transactionIndexRepository.AddAsync(transaction_100);
await _transactionIndexRepository.AddAsync(transaction_101);
await _transactionIndexRepository.AddAsync(transaction_103);
await _transactionIndexRepository.AddAsync(transaction_110);

List<long> inputs = new List<long>()
{
101,
103
};

// var predicates = inputs
// .Select(s => (Expression<Func<TransactionIndex, bool>>)(info => info.LogEvents.Any(x => x.BlockHeight == s)))
// .Aggregate((prev, next) => prev.Or(next));

Expression<Func<TransactionIndex, bool>> mustQuery = item =>
item.LogEvents.Any(x => inputs.Contains(x.BlockHeight));

var queryable = await _transactionIndexRepository.GetQueryableAsync();
// var filterList = queryable.Where(predicates).ToList();
var filterList = queryable.Where(mustQuery).ToList();
filterList.Count.ShouldBe(2);
}

[Fact]
public async Task SubObjectQueryTest()
{
Expand Down

0 comments on commit 35f6bc6

Please sign in to comment.