Skip to content

Commit

Permalink
search improvements
Browse files Browse the repository at this point in the history
  • Loading branch information
uleus authored and uleus committed Feb 4, 2021
1 parent 961e66f commit 49bd878
Show file tree
Hide file tree
Showing 11 changed files with 195 additions and 51 deletions.
11 changes: 8 additions & 3 deletions Elasticsearch/Search.Elasticsearch/Indexing/IndexBaseItem.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,19 +50,24 @@ protected static IAnalysis InitCommonAnalyzers(AnalysisDescriptor analysis)
.Custom("autocomplete", ca => ca
.Tokenizer("autocomplete")
.Filters("stopwords_eng", "trim", "lowercase")
)
)
.Custom("autocomplete_search", ca => ca
.Tokenizer("standard")
.Filters("stopwords_eng", "trim", "lowercase")
)
)
.Custom("keyword_list_serach", ca => ca
.Tokenizer("split_list")
.Filters("stopwords_eng", "trim")
)
)
.Tokenizers(tdesc => tdesc
.EdgeNGram("autocomplete", e => e
.MinGram(3)
.MaxGram(15)
.TokenChars(TokenChar.Letter, TokenChar.Digit)
)
)
.Pattern("split_list", e => e.Pattern(","))
)
.TokenFilters(f => f
.Stop("stopwords_eng", lang => lang
.StopWords("_english_")
Expand Down
27 changes: 22 additions & 5 deletions Elasticsearch/Search.Elasticsearch/Indexing/IndexManagementItem.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,30 @@ public class IndexManagementItem :

public override async Task<CreateIndexResponse> CreateIndex(IElasticClient aClient)
{
await this.DeleteIndex(aClient);
await this.DeleteIndex(aClient);

var res = await aClient.Indices.CreateAsync(Name, i => i
.Settings(InitCommonIndexSettingsDescriptor)
.Map(m => m.AutoMap<SearchableManagementItem>())
);

.Settings(InitCommonIndexSettingsDescriptor)
.Map(m => m
.AutoMap<SearchableManagementItem>()
.Properties(ps => ps
.Text(p => p
.Name("Market")
.Fields(fs => fs
.Text(f => f
.Name("phrase")
.Analyzer("english")
)
.Keyword(f => f
.Name("keyword")
)
)
.Analyzer("autocomplete")
.SearchAnalyzer("autocomplete_search")
)
)
)
);
return res;
}

Expand Down
31 changes: 30 additions & 1 deletion Elasticsearch/Search.Elasticsearch/Indexing/IndexPropertyItem.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,36 @@ public override async Task<CreateIndexResponse> CreateIndex(IElasticClient aClie

var res = await aClient.Indices.CreateAsync(Name, i => i
.Settings(InitCommonIndexSettingsDescriptor)
.Map(m => m.AutoMap<SearchablePropertyItem>())
.Map(m => m
.AutoMap<SearchablePropertyItem>()
.Properties( ps => ps
.Text(p => p
.Name("Market")
.Fields(fs => fs
.Text(f => f
.Name("phrase")
.Analyzer("english")
)
.Keyword(f => f
.Name("keyword")
)
)
.Analyzer("autocomplete")
.SearchAnalyzer("autocomplete_search")
)
.Text( p => p
.Name("City")
.Fields( fs => fs
.Text(f => f
.Name("phrase")
.Analyzer("english")
)
)
.Analyzer("autocomplete")
.SearchAnalyzer("autocomplete_search")
)
)
)
);

return res;
Expand Down
14 changes: 11 additions & 3 deletions Elasticsearch/Search.Elasticsearch/Mapping/SearchableBaseItem.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,22 @@ public class SearchableBaseItem
[Keyword(Name = nameof(Id))]
public int Id { get; set; }

[Keyword(Name = nameof(TypeName))]
public string TypeName { get; protected set; }

[Text(Analyzer = "autocomplete", SearchAnalyzer = "autocomplete_search", Name = nameof(Name))]
public string Name { get; set; }

[Text(Analyzer = "autocomplete", SearchAnalyzer = "autocomplete_search", Name = nameof(Market))]
[Keyword(Name = nameof(State))]
public string State { get; set; }

//multifieds : mapping in code
[Text(Name = nameof(Market))]
public string Market { get; set; }

[Ignore]
public virtual string AdditionalInfo { get; }

public string State { get; set; }

public SearchableBaseItem()
{
}
Expand Down
9 changes: 7 additions & 2 deletions Elasticsearch/Search.Elasticsearch/Mapping/SearchableManagementItem.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,14 @@

namespace Search.Elasticsearch.Mapping
{
[ElasticsearchType(IdProperty = nameof(SearchableBaseItem.Id), RelationName = SearchableManagementItem.TypeName)]
[ElasticsearchType(IdProperty = nameof(SearchableBaseItem.Id), RelationName = SearchableManagementItem.TypeNameDef)]
public class SearchableManagementItem : SearchableBaseItem
{
public const string TypeName = "searchablemanagementitem";
public const string TypeNameDef = "searchablemanagementitem";

public SearchableManagementItem()
{
this.TypeName = TypeNameDef;
}
}
}
15 changes: 12 additions & 3 deletions Elasticsearch/Search.Elasticsearch/Mapping/SearchablePropertyItem.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,31 @@

namespace Search.Elasticsearch.Mapping
{
[ElasticsearchType(IdProperty = nameof(SearchableBaseItem.Id), RelationName = SearchablePropertyItem.TypeName)]
[ElasticsearchType(IdProperty = nameof(SearchableBaseItem.Id), RelationName = SearchablePropertyItem.TypeNameDef)]
public class SearchablePropertyItem : SearchableBaseItem
{
public const string TypeName = "searchablepropertyitem";
public const string TypeNameDef = "searchablepropertyitem";

[Text(Analyzer = "autocomplete", SearchAnalyzer = "autocomplete_search", Name = nameof(FormerName))]
public string FormerName { get; set; }

[Text(Analyzer = "autocomplete", SearchAnalyzer = "autocomplete_search", Name = nameof(StreetAddres))]
public string StreetAddres { get; set; }

[Text(Analyzer = "autocomplete", SearchAnalyzer = "autocomplete_search", Name = nameof(City))]
//multifieds : mapping in code
[Text(Name = nameof(City))]
public string City { get; set; }

public float Lat { get; set; }

public float Lng { get; set; }

public override string AdditionalInfo { get { return $"FormerName:{this.FormerName} City:{this.City} Street:{this.StreetAddres}"; } }

public SearchablePropertyItem()
{
this.TypeName = TypeNameDef;
}

}
}
3 changes: 2 additions & 1 deletion Elasticsearch/Search.Elasticsearch/Search.Elasticsearch.csproj
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="NEST" Version="7.10.1" />
<PackageReference Include="RestSharp" Version="106.11.7" />
</ItemGroup>

</Project>
118 changes: 94 additions & 24 deletions Elasticsearch/Search.Elasticsearch/Search/SearchService.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using Nest;
using RestSharp;
using Search.Elasticsearch.Mapping;
using System.Collections.Generic;
using System.Threading.Tasks;
Expand All @@ -25,45 +26,114 @@ public async Task<SimpleSerachResponse> Search(SimpleSearchRequest aSearchReques
BoolQuery filterQuery = new BoolQuery();
if (!string.IsNullOrEmpty(aSearchRequest.MarketFilterQuery))
{
//we assume that:
// 1: the MarketFilterQuery is in fromat Market1,Market2,Market3 e.g. Austin, San Paulo
// 2: the market names are coming from predefined table, so no spelling mistakes, no autocompletition
// 3: keyword field, and keyword_list_serach analyzer are created with the index
var filterQueryParts = new List<QueryContainer>();
filterQueryParts.Add(
new MatchQuery()
{
Field = $"{nameof(SearchableBaseItem.Market)}",
Query = aSearchRequest.MarketFilterQuery.ToLower(),
Fuzziness = Fuzziness.Auto
}
Field = "Market.keyword",
Query = aSearchRequest.MarketFilterQuery,//.ToLower(),
Analyzer = "keyword_list_serach"
}
);

filterQuery.Filter = filterQueryParts;
}

var results = await _client.SearchAsync<SearchableBaseItem>(s => s
//check the API to be able to get the specific SearchableBaseItem derived types (something similar to ConcreteTypeSelector?)
//for now we use JsonObject and convert it to specific object base on the "TypeName"

var results = await _client.SearchAsync<JsonObject>(s => s
.Size(aSearchRequest.PageSize)
.Skip(aSearchRequest.PageStartIndex)
.Index(Indices.Index(aSearchRequest.Indices))
.Query(q => q
.MultiMatch(m => m
.Query(aSearchRequest.AllStringFiledsQuery.ToLower())
.Fields(ff => ff
.Field($"{nameof(SearchableBaseItem.Name)}")
.Field($"{nameof(SearchableBaseItem.Market)}")
.Field($"{nameof(SearchableBaseItem.State)}")
.Field($"{nameof(SearchablePropertyItem.FormerName)}")
.Field($"{nameof(SearchablePropertyItem.StreetAddres)}")
.Field($"{nameof(SearchablePropertyItem.City)}")
)
//.Fuzziness(Fuzziness.Auto)
)
&& filterQuery
)
);
.Query(q => q
.Bool(b => b
.Should(
// for City and Market a 'phrase' field is crated, which allowes
// to better place the item in result when the city/market was put in the AllStringFiledsQuery
bs => bs.MatchPhrase(x => x
.Query(aSearchRequest.AllStringFiledsQuery.ToLower())
.Field("City.phrase")
),
bs => bs.MatchPhrase(x => x
.Query(aSearchRequest.AllStringFiledsQuery.ToLower())
.Field("Market.phrase")
)
)
.Must(
// we search all text fields with the AllStringFiledsQuery
// Name, Market, FormerName, StreetAdress, City text field uses:
// 'autocomplete' analyzer when indexing data
// 'autocomplete_search' analyzer when searching
// both analyzers are crated on index creation
bs => bs.MultiMatch( m => m
.Query(aSearchRequest.AllStringFiledsQuery.ToLower())
.Fields(ff => ff
.Field($"{nameof(SearchableBaseItem.Name)}")
.Field($"{nameof(SearchableBaseItem.Market)}")
.Field($"{nameof(SearchableBaseItem.State)}")
.Field($"{nameof(SearchablePropertyItem.FormerName)}")
.Field($"{nameof(SearchablePropertyItem.StreetAddres)}")
.Field($"{nameof(SearchablePropertyItem.City)}")
)
.Fuzziness(Fuzziness.Auto)
),
bs => filterQuery
)
)
)
);



return new SimpleSerachResponse()
{
{
TotalItems = results.Total,
Items = results.Documents
Items = ConvertResults(results)
};
}
}


private List<SearchableBaseItem> ConvertResults(ISearchResponse<JsonObject> aSerachResponse)
{

List<SearchableBaseItem> ret = new List<SearchableBaseItem>();

foreach (var item in aSerachResponse.Documents)
{
switch (item["TypeName"])
{
case "searchablemanagementitem":
ret.Add(new SearchableManagementItem()
{
Id = int.Parse(item["Id"].ToString()),
Name = (string)item["Name"],
State = (string)item["State"],
Market = (string)item["Market"]
});
break;
case "searchablepropertyitem":
ret.Add(new SearchablePropertyItem()
{
Id = int.Parse(item["Id"].ToString()),
Name = item["Name"].ToString(),
State = item["State"].ToString(),
Market = item["Market"].ToString(),
FormerName = item.ContainsKey("FormerName") ? item["FormerName"].ToString() : "",
StreetAddres = item["StreetAddres"].ToString(),
City = (string)item["City"].ToString(),
//Lat = (float) item["lat"],
//Lng = (float) item["lng"]
});
break;
}
}

return ret;
}
}
}
12 changes: 6 additions & 6 deletions Elasticsearch/Search.IndexData.Exe/DataProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,16 @@ public DataProvider()
Market="Austin",
State ="GS",
StreetAddres = "3549 Curry Lane",
City = "Marietta AAA"
City = "San Francesco"
},
new SearchablePropertyItem()
{
Id = 2,
Name ="Forest at Columbia AAAA",
StreetAddres = "1000 Merrick Ferry Road",
Name ="Forest at Columbia AAAA San in Paulo",
StreetAddres = "1000 Merrick Ferry Road San in Paulo",
Market="Austin",
State ="GS",
City = "Marietta ZZZ"
City = "San in Paulo"
},
new SearchablePropertyItem()
{
Expand All @@ -41,7 +41,7 @@ public DataProvider()
StreetAddres = "nowa",
Market="1000 Bells Ferry Road",
State ="TX",
City = "Marietta"
City = "San Paulo"
},
};

Expand All @@ -51,7 +51,7 @@ public DataProvider()
{
Id = 1,
Name = "Holland Residential",
Market="San Paulo",
Market="San Paulo AAA",
State ="TX"
}
};
Expand Down
2 changes: 1 addition & 1 deletion Elasticsearch/Search.IndexData.Exe/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ static void Main(string[] args)
//{
// Console.WriteLine(ex);
//}

//try
//{
// dataProvider.LoadProperties(@"D:\sandbox-private\GitHub\webapi\Elasticsearch\properties.json");
Expand Down
Loading

0 comments on commit 49bd878

Please sign in to comment.