diff --git a/Elasticsearch/Search.Elasticsearch/Indexing/IndexBaseItem.cs b/Elasticsearch/Search.Elasticsearch/Indexing/IndexBaseItem.cs index 67ebb16..0ea8637 100644 --- a/Elasticsearch/Search.Elasticsearch/Indexing/IndexBaseItem.cs +++ b/Elasticsearch/Search.Elasticsearch/Indexing/IndexBaseItem.cs @@ -41,41 +41,34 @@ protected static IPromise InitCommonIndexSettingsDescriptor(Inde return aDescriptor .NumberOfReplicas(0) .NumberOfShards(1) - /* .Analysis(InitCommonAnalyzers)*/; + .Analysis(InitCommonAnalyzers); } - //protected static IAnalysis InitCommonAnalyzers(AnalysisDescriptor analysis) - //{ - // return analysis.Analyzers(a => a - // .Custom("html_stripper", cc => cc - // .Filters("eng_stopwords", "trim", "lowercase") - // .CharFilters("html_strip") - // .Tokenizer("autocomplete") - // ) - // .Custom("keywords_wo_stopwords", cc => cc - // .Filters("eng_stopwords", "trim", "lowercase") - // .CharFilters("html_strip") - // .Tokenizer("key_tokenizer") - // ) - // .Custom("autocomplete", cc => cc - // .Filters("eng_stopwords", "trim", "lowercase") - // .Tokenizer("autocomplete") - // ) - // ) - // .Tokenizers(tdesc => tdesc - // .Keyword("key_tokenizer", t => t) - // .EdgeNGram("autocomplete", e => e - // .MinGram(3) - // .MaxGram(15) - // .TokenChars(TokenChar.Letter, TokenChar.Digit) - // ) - // ) - // .TokenFilters(f => f - // .Stop("eng_stopwords", lang => lang - // .StopWords("_english_") - // ) - // ); - //} + protected static IAnalysis InitCommonAnalyzers(AnalysisDescriptor analysis) + { + return analysis.Analyzers(a => a + .Custom("autocomplete", ca => ca + .Filters("eng_stopwords", "trim", "lowercase") + .Tokenizer("autocomplete") + ) + .Custom("autocomplete_search", ca => ca + .Filters("eng_stopwords", "trim", "lowercase") + .Tokenizer("lowercase") + ) + ) + .Tokenizers(tdesc => tdesc + .EdgeNGram("autocomplete", e => e + .MinGram(3) + .MaxGram(15) + .TokenChars(TokenChar.Letter, TokenChar.Digit) + ) + ) + .TokenFilters(f => f + .Stop("eng_stopwords", lang => lang + .StopWords("_english_") + ) + ); + } } } diff --git a/Elasticsearch/Search.Elasticsearch/Mapping/SearchableBaseItem.cs b/Elasticsearch/Search.Elasticsearch/Mapping/SearchableBaseItem.cs index 8db2f54..73e392e 100644 --- a/Elasticsearch/Search.Elasticsearch/Mapping/SearchableBaseItem.cs +++ b/Elasticsearch/Search.Elasticsearch/Mapping/SearchableBaseItem.cs @@ -7,13 +7,13 @@ public class SearchableBaseItem [Keyword(Name = nameof(Id))] public int Id { get; set; } - [Text(/*Analyzer = "autocomplete", */Name = nameof(Name))] + [Text(Analyzer = "autocomplete", SearchAnalyzer = "autocomplete_search", Name = nameof(Name))] public string Name { get; set; } - [Text(/*Analyzer = "autocomplete", */Name = nameof(Market))] + [Text(/*Analyzer = "autocomplete",*/ Name = nameof(Market))] public string Market { get; set; } - [Text(/*Analyzer = "autocomplete", */Name = nameof(State))] + [Text(/*Analyzer = "autocomplete",*/ Name = nameof(State))] public string State { get; set; } public SearchableBaseItem() diff --git a/Elasticsearch/Search.Elasticsearch/Mapping/SearchablePropertyItem.cs b/Elasticsearch/Search.Elasticsearch/Mapping/SearchablePropertyItem.cs index 3fd558f..cd0881c 100644 --- a/Elasticsearch/Search.Elasticsearch/Mapping/SearchablePropertyItem.cs +++ b/Elasticsearch/Search.Elasticsearch/Mapping/SearchablePropertyItem.cs @@ -16,10 +16,10 @@ public class SearchablePropertyItem : SearchableBaseItem [Text(/*Analyzer = "autocomplete", */Name = nameof(City))] public string City { get; set; } - [Text(Name=nameof(Lat))] - public string Lat { get; set; } + //[Text(Name=nameof(Lat))] + public float Lat { get; set; } - [Text(Name = nameof(Lng))] - public string Lng { get; set; } + //[Text(Name = nameof(Lng))] + public float Lng { get; set; } } } diff --git a/Elasticsearch/Search.Elasticsearch/Search/SearchService.cs b/Elasticsearch/Search.Elasticsearch/Search/SearchService.cs index d5f1b11..e628814 100644 --- a/Elasticsearch/Search.Elasticsearch/Search/SearchService.cs +++ b/Elasticsearch/Search.Elasticsearch/Search/SearchService.cs @@ -7,13 +7,12 @@ namespace Search.Elasticsearch.Search { public interface ISearchService { - Task> Search(SimpleSearchRequest aSearchRequest); + Task Search(SimpleSearchRequest aSearchRequest); } public class SearchService : ISearchService { - private IElasticClient _client; public SearchService(IElasticClientFactoryService aElasticClientFactoryService) @@ -21,17 +20,19 @@ public SearchService(IElasticClientFactoryService aElasticClientFactoryService) _client = aElasticClientFactoryService.CreateElasticClient(); } - public async Task> Search(SimpleSearchRequest aSearchRequest) + public async Task Search(SimpleSearchRequest aSearchRequest) { BoolQuery filterQuery = new BoolQuery(); if (!string.IsNullOrEmpty(aSearchRequest.Filter)) { var filterQueryParts = new List(); - filterQueryParts.Add(new MatchQuery() - { - Field = $"{nameof(SearchableBaseItem.Market)}", - Query = aSearchRequest.Filter - } + filterQueryParts.Add( + new MatchQuery() + { + Field = $"{nameof(SearchableBaseItem.Market)}", + Query = aSearchRequest.Filter.ToLower(), + Fuzziness = Fuzziness.Auto + } ); filterQuery.Filter = filterQueryParts; @@ -41,10 +42,11 @@ public async Task> Search(SimpleSearchReques .Size(aSearchRequest.PageSize) .Skip(aSearchRequest.PageStartIndex) .Index(Indices.Index(aSearchRequest.Indices)) + .Query(q => q .MultiMatch(m => m - .Query(aSearchRequest.Query) - .Fields(ff => ff + .Query(aSearchRequest.Query.ToLower()) + .Fields(ff => ff .Field($"{nameof(SearchableBaseItem.Name)}") .Field($"{nameof(SearchableBaseItem.Market)}") .Field($"{nameof(SearchableBaseItem.State)}") @@ -52,12 +54,17 @@ public async Task> Search(SimpleSearchReques .Field($"{nameof(SearchablePropertyItem.StreetAddres)}") .Field($"{nameof(SearchablePropertyItem.City)}") ) - ) + //.Fuzziness(Fuzziness.Auto) + ) && filterQuery - ) + ) ); - return results; - } + return new SimpleSerachResponse() + { + TotalItems = results.Total, + Items = results.Documents + }; + } } } diff --git a/Elasticsearch/Search.Elasticsearch/Search/SearchRequest.cs b/Elasticsearch/Search.Elasticsearch/Search/SimpleSearchRequest.cs similarity index 100% rename from Elasticsearch/Search.Elasticsearch/Search/SearchRequest.cs rename to Elasticsearch/Search.Elasticsearch/Search/SimpleSearchRequest.cs diff --git a/Elasticsearch/Search.Elasticsearch/Search/SimpleSerachResponse.cs b/Elasticsearch/Search.Elasticsearch/Search/SimpleSerachResponse.cs new file mode 100644 index 0000000..ceac312 --- /dev/null +++ b/Elasticsearch/Search.Elasticsearch/Search/SimpleSerachResponse.cs @@ -0,0 +1,13 @@ +using Search.Elasticsearch.Mapping; +using System; +using System.Collections.Generic; +using System.Text; + +namespace Search.Elasticsearch.Search +{ + public class SimpleSerachResponse + { + public IReadOnlyCollection Items { get; set; } + public long TotalItems { get; set; } + } +} diff --git a/Elasticsearch/Search.IndexData.Exe/DataProvider.cs b/Elasticsearch/Search.IndexData.Exe/DataProvider.cs index 1f8bd07..be7a021 100644 --- a/Elasticsearch/Search.IndexData.Exe/DataProvider.cs +++ b/Elasticsearch/Search.IndexData.Exe/DataProvider.cs @@ -1,11 +1,14 @@ -using Newtonsoft.Json; +using Nancy.Json; +using Newtonsoft.Json; using RestSharp; using Search.Elasticsearch.Indexing; using Search.Elasticsearch.Mapping; +using Search.WebAPI.Exe.Dto; using System; using System.Collections.Generic; using System.IO; + namespace Search.IndexData.Exe { class DataProvider : @@ -14,42 +17,33 @@ class DataProvider : public List Properties { get; protected set; } public List Managements { get; protected set; } - public DataProvider(string aPropertiesJsonFilePath, string aMgmtJsonFilePath) - { - //List data = JsonConvert.DeserializeObject>(File.ReadAllText(aPropertiesJsonFilePath), new JsonSerializerSettings() - //{ - // Culture = global.C - //}); - //new JsonSerializerSettings() - //{ - // FloatFormatHandling = FloatFormatHandling. - //}); - + public DataProvider() + { this.Properties = new List() { new SearchablePropertyItem() { Id= 1, - Name ="prop1", + Name ="My granny has a wooden super chair", StreetAddres = "test", - Market="Warszawa", - State ="a" + Market="Austin", + State ="GS" }, new SearchablePropertyItem() { Id = 2, Name ="prop2", StreetAddres = "nowa", - Market="Warszawa", - State ="a" + Market="Austin", + State ="GS" }, new SearchablePropertyItem() { Id = 3, Name ="prop3", StreetAddres = "nowa", - Market="Szczecin", - State ="a" + Market="San Francisco", + State ="CA" }, }; @@ -58,11 +52,96 @@ public DataProvider(string aPropertiesJsonFilePath, string aMgmtJsonFilePath) new SearchableManagementItem() { Id = 1, - Name = "test", - Market="Szczecin", - State ="a" + Name = "Company A", + Market="San Paulo", + State ="TX" } - }; + }; + + } + + public int LoadManagemetns(string aPath) + { + int iErrorCounter = 0; + var data = System.Text.Json.JsonSerializer.Deserialize>(File.ReadAllText(aPath)); + + foreach (var mw in data) + if (mw.mgmt != null) + this.Managements.Add( + new SearchableManagementItem() + { + Id = mw.mgmt.mgmtID, + Name = mw.mgmt.name, + State = mw.mgmt.state, + Market = mw.mgmt.market + } + ); + else + { + iErrorCounter++; + } + + return iErrorCounter; + } + + public void LoadProperties(string aPath) + { + int iErrorCounter = 0; + var data = System.Text.Json.JsonSerializer.Deserialize>(File.ReadAllText(aPath)); + + foreach (var pw in data) + if (pw.property != null) + this.Properties.Add( + new SearchablePropertyItem() + { + Id = pw.property.propertyID, + Name = pw.property.name, + State = pw.property.state, + Market = pw.property.market, + FormerName = pw.property.formerName, + City = pw.property.city, + StreetAddres = pw.property.streetAddres, + Lat = pw.property.lat, + Lng = pw.property.lng + } + ); + else + { + iErrorCounter++; + } + + + } + + class PropertyItemWrapper + { + public PropertyItem property { get; set; } + } + + public class PropertyItem + { + public int propertyID { get; set; } + public string name { get; set; } + public string formerName { get; set; } + public string streetAddres { get; set; } + public string city { get; set; } + public string market { get; set; } + public string state { get; set; } + public float lat { get; set; } + public float lng { get; set; } + } + + class ManagementItemWrapper + { + public ManagementItem mgmt { get; set; } + } + + public class ManagementItem + { + public int mgmtID { get; set; } + public string name { get; set; } + public string market { get; set; } + public string state { get; set; } } } } diff --git a/Elasticsearch/Search.IndexData.Exe/Program.cs b/Elasticsearch/Search.IndexData.Exe/Program.cs index e2d1a99..16f524b 100644 --- a/Elasticsearch/Search.IndexData.Exe/Program.cs +++ b/Elasticsearch/Search.IndexData.Exe/Program.cs @@ -1,6 +1,13 @@ using Nest; +using Newtonsoft.Json; using Search.Elasticsearch.Indexing; +using Search.Elasticsearch.Mapping; +using Search.WebAPI.Exe.Dto; using System; +using System.Collections.Generic; +using System.IO; +using System.Text.Json; +using static Search.IndexData.Exe.DataProvider; namespace Search.IndexData.Exe { @@ -12,16 +19,30 @@ class Program static void Main(string[] args) { - var node = new Uri(AWSESUri); var settings = new ConnectionSettings(node).BasicAuthentication(AWSESUser, AWSESPwd); settings.DisableDirectStreaming(); var client = new ElasticClient(settings); - DataProvider dataProvider = new DataProvider( - @"D:\sandbox-private\GitHub\webapi\Elasticsearch\properties.json", - @"D:\sandbox-private\GitHub\webapi\Elasticsearch\mgmt.json"); + DataProvider dataProvider = new DataProvider(); + //try + //{ + // dataProvider.LoadManagemetns(@"D:\sandbox-private\GitHub\webapi\Elasticsearch\mgmt.json"); + //} + //catch (Exception ex) + //{ + // Console.WriteLine(ex); + //} + + //try + //{ + // dataProvider.LoadProperties(@"D:\sandbox-private\GitHub\webapi\Elasticsearch\properties.json"); + //} + //catch (Exception ex) + //{ + // Console.WriteLine(ex); + //} IndexPropertyItem indexPropertyItem = new IndexPropertyItem(); var resCreate = indexPropertyItem.CreateIndex(client).Result; @@ -31,12 +52,6 @@ static void Main(string[] args) resCreate = indexManagementItem.CreateIndex(client).Result; resInsert = indexManagementItem.InsertDataIntoIndex(client, dataProvider).Result; - //SearchService searchService = new SearchService(client); - //searchService.Search(new CommonSearchRequest() - //{ - // Query = "test" - //}, new List() { propertyIndexer, managementIndex }); - } } } diff --git a/Elasticsearch/Search.WebAPI.Exe/Controllers/SearchController.cs b/Elasticsearch/Search.WebAPI.Exe/Controllers/SearchController.cs index 5b6a35a..7c8a1e0 100644 --- a/Elasticsearch/Search.WebAPI.Exe/Controllers/SearchController.cs +++ b/Elasticsearch/Search.WebAPI.Exe/Controllers/SearchController.cs @@ -4,6 +4,7 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using Search.Elasticsearch; +using Search.Elasticsearch.Mapping; using Search.Elasticsearch.Search; using Search.WebAPI.Exe.Dto; @@ -23,7 +24,7 @@ public SearchController(ILogger aLogger, ISearchService aSerac _searchSevice = aSerachService; } - [HttpGet("{aQuery}")] + [HttpGet] public async Task Search(SearchCriteriaDto aRequest) { _logger?.LogDebug("'{0}' has been invoked", nameof(SearchController)); @@ -32,16 +33,29 @@ public async Task Search(SearchCriteriaDto aRequest) try { var result = await _searchSevice.Search(new SimpleSearchRequest() - { - PageSize = aRequest.PageSize, - PageStartIndex = aRequest.PageStartIndex, - Query = aRequest.Phase, - Filter = aRequest.Market, - Indices = new List() { Config.IndexPropertyItemName, Config.IndexManagementItemName } - } - ); - - response.Model = result.Documents; + { + PageSize = aRequest.PageSize, + PageStartIndex = aRequest.PageStartIndex, + Query = aRequest.Phase, + QueryFields = new List() + { + nameof(SearchableBaseItem.Name), + nameof(SearchableBaseItem.Market), + nameof(SearchableBaseItem.State), + nameof(SearchablePropertyItem.FormerName), + nameof(SearchablePropertyItem.StreetAddres), + nameof(SearchablePropertyItem.City) + }, + Filter = aRequest.Market, + FilterFields = new List() { nameof(SearchableBaseItem.Market) }, + Indices = new List() { Config.IndexPropertyItemName, Config.IndexManagementItemName } + } + ); + + response.Model = result.Items; + response.ItemsCount = result.TotalItems; + response.PageSize = aRequest.PageSize; + response.PageNumber = aRequest.PageStartIndex / aRequest.PageSize; _logger?.LogInformation("Trasnfer '{0}' has been started", response.Model); } diff --git a/Elasticsearch/Search.WebAPI.Exe/Services/ElasticClientFactoryService.cs b/Elasticsearch/Search.WebAPI.Exe/Services/ElasticClientFactoryService.cs index 8dd1661..20e4ae9 100644 --- a/Elasticsearch/Search.WebAPI.Exe/Services/ElasticClientFactoryService.cs +++ b/Elasticsearch/Search.WebAPI.Exe/Services/ElasticClientFactoryService.cs @@ -21,6 +21,7 @@ public IElasticClient CreateElasticClient() var node = new Uri(_configuration["AWSES:URL"]); var settings = new ConnectionSettings(node) .BasicAuthentication(_configuration["AWSES:User"], _configuration["AWSES:Pwd"]); + settings.ThrowExceptions(alwaysThrow: true); settings.DisableDirectStreaming(); return new ElasticClient(settings); diff --git a/Elasticsearch/Search.WebAPI/Responses.cs b/Elasticsearch/Search.WebAPI/Responses.cs index d0fc603..b7f46d7 100644 --- a/Elasticsearch/Search.WebAPI/Responses.cs +++ b/Elasticsearch/Search.WebAPI/Responses.cs @@ -23,7 +23,7 @@ public interface IListResponse : IResponse public interface IPagedResponse : IListResponse { - int ItemsCount { get; set; } + long ItemsCount { get; set; } double PageCount { get; } } @@ -70,11 +70,11 @@ public class PagedResponse : IPagedResponse public IEnumerable Model { get; set; } - public int PageSize { get; set; } + public long PageSize { get; set; } - public int PageNumber { get; set; } + public long PageNumber { get; set; } - public int ItemsCount { get; set; } + public long ItemsCount { get; set; } public double PageCount => ItemsCount < PageSize ? 1 : (int)(((double)ItemsCount / PageSize) + 1);