From e21c9a4e6316689844a2c2e60f5cb092698367a7 Mon Sep 17 00:00:00 2001 From: Zachary Vonler Date: Tue, 17 Oct 2023 15:31:30 -0500 Subject: [PATCH 01/16] Added search using qualifier[:=]value syntax. --- commands/lib/search.go | 137 +++++++++++++++++++++++++++++++++--- commands/lib/search_test.go | 24 +++++++ 2 files changed, 151 insertions(+), 10 deletions(-) diff --git a/commands/lib/search.go b/commands/lib/search.go index 5e44e2273e1..3aacefcc79f 100644 --- a/commands/lib/search.go +++ b/commands/lib/search.go @@ -38,24 +38,141 @@ func LibrarySearch(ctx context.Context, req *rpc.LibrarySearchRequest) (*rpc.Lib return searchLibrary(req, lm), nil } +// MatcherTokensFromQueryString parses the query string into tokens of interest +// for the qualifier-value pattern matching. +func MatcherTokensFromQueryString(query string) []string { + escaped := false + quoted := false + tokens := []string{} + sb := &strings.Builder{} + + for _, r := range query { + // Short circuit the loop on backslash so that all other paths can clear + // the escaped flag. + if !escaped && r == '\\' { + escaped = true + continue + } + + if r == '"' { + if !escaped { + quoted = !quoted + } else { + sb.WriteRune(r) + } + } else if !quoted && r == ' ' { + tokens = append(tokens, strings.ToLower(sb.String())) + sb.Reset() + } else { + sb.WriteRune(r) + } + escaped = false + } + if sb.Len() > 0 { + tokens = append(tokens, strings.ToLower(sb.String())) + } + + return tokens +} + +// DefaulLibraryMatchExtractor returns a string describing the library that +// is used for the simple search. +func DefaultLibraryMatchExtractor(lib *librariesindex.Library) string { + res := lib.Name + " " + + lib.Latest.Paragraph + " " + + lib.Latest.Sentence + " " + + lib.Latest.Author + " " + for _, include := range lib.Latest.ProvidesIncludes { + res += include + " " + } + return res +} + +// MatcherFromQueryString returns a closure that takes a library as a +// parameter and returns true if the library matches the query. +func MatcherFromQueryString(query string) func(*librariesindex.Library) bool { + // A qv-query is one using [:=] syntax. + qvQuery := strings.Contains(query, ":") || strings.Contains(query, "=") + + if !qvQuery { + queryTerms := utils.SearchTermsFromQueryString(query) + return func(lib *librariesindex.Library) bool { + return utils.Match(DefaultLibraryMatchExtractor(lib), queryTerms) + } + } + + joinedStrings := func(strs []string) string { + return strings.Join(strs, " ") + } + + qualifiers := []struct { + key string + extractor func(*librariesindex.Library) string + }{ + // The library name comes from the Library object. + {"name", func(lib *librariesindex.Library) string { return lib.Name }}, + + // All other values come from the latest Release. + {"architectures", func(lib *librariesindex.Library) string { return joinedStrings(lib.Latest.Architectures) }}, + {"author", func(lib *librariesindex.Library) string { return lib.Latest.Author }}, + {"category", func(lib *librariesindex.Library) string { return lib.Latest.Category }}, + {"dependencies", func(lib *librariesindex.Library) string { + names := []string{} + for _, dep := range lib.Latest.Dependencies { + names = append(names, dep.GetName()) + } + return joinedStrings(names) + }}, + {"maintainer", func(lib *librariesindex.Library) string { return lib.Latest.Maintainer }}, + {"paragraph", func(lib *librariesindex.Library) string { return lib.Latest.Paragraph }}, + {"sentence", func(lib *librariesindex.Library) string { return lib.Latest.Sentence }}, + {"types", func(lib *librariesindex.Library) string { return joinedStrings(lib.Latest.Types) }}, + {"version", func(lib *librariesindex.Library) string { return lib.Latest.Version.String() }}, + {"website", func(lib *librariesindex.Library) string { return lib.Latest.Website }}, + } + + queryTerms := MatcherTokensFromQueryString(query) + + return func(lib *librariesindex.Library) bool { + matched := true + for _, term := range queryTerms { + + // Flag indicating whether the search term matched a known qualifier + knownQualifier := false + + for _, q := range qualifiers { + if strings.HasPrefix(term, q.key+":") { + target := strings.TrimPrefix(term, q.key+":") + matched = (matched && utils.Match(q.extractor(lib), []string{target})) + knownQualifier = true + break + } else if strings.HasPrefix(term, q.key+"=") { + target := strings.TrimPrefix(term, q.key+"=") + matched = (matched && strings.ToLower(q.extractor(lib)) == target) + knownQualifier = true + break + } + } + + if !knownQualifier { + matched = (matched && utils.Match(DefaultLibraryMatchExtractor(lib), []string{term})) + } + } + return matched + } +} + func searchLibrary(req *rpc.LibrarySearchRequest, lm *librariesmanager.LibrariesManager) *rpc.LibrarySearchResponse { res := []*rpc.SearchedLibrary{} query := req.GetSearchArgs() if query == "" { query = req.GetQuery() } - queryTerms := utils.SearchTermsFromQueryString(query) - for _, lib := range lm.Index.Libraries { - toTest := lib.Name + " " + - lib.Latest.Paragraph + " " + - lib.Latest.Sentence + " " + - lib.Latest.Author + " " - for _, include := range lib.Latest.ProvidesIncludes { - toTest += include + " " - } + matcher := MatcherFromQueryString(query) - if utils.Match(toTest, queryTerms) { + for _, lib := range lm.Index.Libraries { + if matcher(lib) { res = append(res, indexLibraryToRPCSearchLibrary(lib, req.GetOmitReleasesDetails())) } } diff --git a/commands/lib/search_test.go b/commands/lib/search_test.go index affb9ba6911..109aa408c7c 100644 --- a/commands/lib/search_test.go +++ b/commands/lib/search_test.go @@ -94,3 +94,27 @@ func TestSearchLibraryFields(t *testing.T) { require.Len(t, res, 19) require.Equal(t, "FlashStorage", res[0]) } + +func TestSearchLibraryWithQualifiers(t *testing.T) { + lm := librariesmanager.NewLibraryManager(fullIndexPath, nil) + lm.LoadIndex() + + query := func(q string) []string { + libs := []string{} + for _, lib := range searchLibrary(&rpc.LibrarySearchRequest{SearchArgs: q}, lm).Libraries { + libs = append(libs, lib.Name) + } + return libs + } + + res := query("name:FlashStorage") + require.Len(t, res, 7) + + res = query("name=FlashStorage") + require.Len(t, res, 1) + require.Equal(t, "FlashStorage", res[0]) + + res = query("name=\"Painless Mesh\"") + require.Len(t, res, 1) + require.Equal(t, "Painless Mesh", res[0]) +} From 21ee5f8143bb6caefbb7fa57907197a8cdb96b71 Mon Sep 17 00:00:00 2001 From: Zachary Vonler Date: Wed, 18 Oct 2023 12:56:48 -0500 Subject: [PATCH 02/16] Lowercased names to prevent export. --- commands/lib/search.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/commands/lib/search.go b/commands/lib/search.go index 3aacefcc79f..d21a23842b6 100644 --- a/commands/lib/search.go +++ b/commands/lib/search.go @@ -38,9 +38,9 @@ func LibrarySearch(ctx context.Context, req *rpc.LibrarySearchRequest) (*rpc.Lib return searchLibrary(req, lm), nil } -// MatcherTokensFromQueryString parses the query string into tokens of interest +// matcherTokensFromQueryString parses the query string into tokens of interest // for the qualifier-value pattern matching. -func MatcherTokensFromQueryString(query string) []string { +func matcherTokensFromQueryString(query string) []string { escaped := false quoted := false tokens := []string{} @@ -75,9 +75,9 @@ func MatcherTokensFromQueryString(query string) []string { return tokens } -// DefaulLibraryMatchExtractor returns a string describing the library that +// defaulLibraryMatchExtractor returns a string describing the library that // is used for the simple search. -func DefaultLibraryMatchExtractor(lib *librariesindex.Library) string { +func defaultLibraryMatchExtractor(lib *librariesindex.Library) string { res := lib.Name + " " + lib.Latest.Paragraph + " " + lib.Latest.Sentence + " " + @@ -88,16 +88,16 @@ func DefaultLibraryMatchExtractor(lib *librariesindex.Library) string { return res } -// MatcherFromQueryString returns a closure that takes a library as a +// matcherFromQueryString returns a closure that takes a library as a // parameter and returns true if the library matches the query. -func MatcherFromQueryString(query string) func(*librariesindex.Library) bool { +func matcherFromQueryString(query string) func(*librariesindex.Library) bool { // A qv-query is one using [:=] syntax. qvQuery := strings.Contains(query, ":") || strings.Contains(query, "=") if !qvQuery { queryTerms := utils.SearchTermsFromQueryString(query) return func(lib *librariesindex.Library) bool { - return utils.Match(DefaultLibraryMatchExtractor(lib), queryTerms) + return utils.Match(defaultLibraryMatchExtractor(lib), queryTerms) } } @@ -131,7 +131,7 @@ func MatcherFromQueryString(query string) func(*librariesindex.Library) bool { {"website", func(lib *librariesindex.Library) string { return lib.Latest.Website }}, } - queryTerms := MatcherTokensFromQueryString(query) + queryTerms := matcherTokensFromQueryString(query) return func(lib *librariesindex.Library) bool { matched := true @@ -155,7 +155,7 @@ func MatcherFromQueryString(query string) func(*librariesindex.Library) bool { } if !knownQualifier { - matched = (matched && utils.Match(DefaultLibraryMatchExtractor(lib), []string{term})) + matched = (matched && utils.Match(defaultLibraryMatchExtractor(lib), []string{term})) } } return matched @@ -169,7 +169,7 @@ func searchLibrary(req *rpc.LibrarySearchRequest, lm *librariesmanager.Libraries query = req.GetQuery() } - matcher := MatcherFromQueryString(query) + matcher := matcherFromQueryString(query) for _, lib := range lm.Index.Libraries { if matcher(lib) { From 5de948d95caaf02f1fe3392e629b23c4cc4ef730 Mon Sep 17 00:00:00 2001 From: Zach Vonler Date: Fri, 20 Oct 2023 09:26:52 -0500 Subject: [PATCH 03/16] Improvement from review Co-authored-by: Alessio Perugini --- commands/lib/search.go | 44 ++++++++++++++++++------------------------ 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/commands/lib/search.go b/commands/lib/search.go index d21a23842b6..bab08f0ca3b 100644 --- a/commands/lib/search.go +++ b/commands/lib/search.go @@ -105,31 +105,25 @@ func matcherFromQueryString(query string) func(*librariesindex.Library) bool { return strings.Join(strs, " ") } - qualifiers := []struct { - key string - extractor func(*librariesindex.Library) string - }{ - // The library name comes from the Library object. - {"name", func(lib *librariesindex.Library) string { return lib.Name }}, - - // All other values come from the latest Release. - {"architectures", func(lib *librariesindex.Library) string { return joinedStrings(lib.Latest.Architectures) }}, - {"author", func(lib *librariesindex.Library) string { return lib.Latest.Author }}, - {"category", func(lib *librariesindex.Library) string { return lib.Latest.Category }}, - {"dependencies", func(lib *librariesindex.Library) string { - names := []string{} - for _, dep := range lib.Latest.Dependencies { - names = append(names, dep.GetName()) - } - return joinedStrings(names) - }}, - {"maintainer", func(lib *librariesindex.Library) string { return lib.Latest.Maintainer }}, - {"paragraph", func(lib *librariesindex.Library) string { return lib.Latest.Paragraph }}, - {"sentence", func(lib *librariesindex.Library) string { return lib.Latest.Sentence }}, - {"types", func(lib *librariesindex.Library) string { return joinedStrings(lib.Latest.Types) }}, - {"version", func(lib *librariesindex.Library) string { return lib.Latest.Version.String() }}, - {"website", func(lib *librariesindex.Library) string { return lib.Latest.Website }}, - } +var qualifiers map[string]func(lib *librariesindex.Library) string = map[string]func(lib *librariesindex.Library) string{ + "name": func(lib *librariesindex.Library) string { return lib.Name }, + "architectures": func(lib *librariesindex.Library) string { return strings.Join(lib.Latest.Architectures, " ") }, + "author": func(lib *librariesindex.Library) string { return lib.Latest.Author }, + "category": func(lib *librariesindex.Library) string { return lib.Latest.Category }, + "dependencies": func(lib *librariesindex.Library) string { + names := make([]string, len(lib.Latest.Dependencies)) + for i, dep := range lib.Latest.Dependencies { + names[i] = dep.GetName() + } + return strings.Join(names, " ") + }, + "maintainer": func(lib *librariesindex.Library) string { return lib.Latest.Maintainer }, + "paragraph": func(lib *librariesindex.Library) string { return lib.Latest.Paragraph }, + "sentence": func(lib *librariesindex.Library) string { return lib.Latest.Sentence }, + "types": func(lib *librariesindex.Library) string { return strings.Join(lib.Latest.Types, " ") }, + "version": func(lib *librariesindex.Library) string { return lib.Latest.Version.String() }, + "website": func(lib *librariesindex.Library) string { return lib.Latest.Website }, +} queryTerms := matcherTokensFromQueryString(query) From 2fd3c30f872466cc53ba46979d4053a5e194d84c Mon Sep 17 00:00:00 2001 From: Zachary Vonler Date: Fri, 20 Oct 2023 09:34:29 -0500 Subject: [PATCH 04/16] Fixes to use map. --- commands/lib/search.go | 44 +++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/commands/lib/search.go b/commands/lib/search.go index bab08f0ca3b..da582dd738f 100644 --- a/commands/lib/search.go +++ b/commands/lib/search.go @@ -88,23 +88,6 @@ func defaultLibraryMatchExtractor(lib *librariesindex.Library) string { return res } -// matcherFromQueryString returns a closure that takes a library as a -// parameter and returns true if the library matches the query. -func matcherFromQueryString(query string) func(*librariesindex.Library) bool { - // A qv-query is one using [:=] syntax. - qvQuery := strings.Contains(query, ":") || strings.Contains(query, "=") - - if !qvQuery { - queryTerms := utils.SearchTermsFromQueryString(query) - return func(lib *librariesindex.Library) bool { - return utils.Match(defaultLibraryMatchExtractor(lib), queryTerms) - } - } - - joinedStrings := func(strs []string) string { - return strings.Join(strs, " ") - } - var qualifiers map[string]func(lib *librariesindex.Library) string = map[string]func(lib *librariesindex.Library) string{ "name": func(lib *librariesindex.Library) string { return lib.Name }, "architectures": func(lib *librariesindex.Library) string { return strings.Join(lib.Latest.Architectures, " ") }, @@ -125,6 +108,19 @@ var qualifiers map[string]func(lib *librariesindex.Library) string = map[string] "website": func(lib *librariesindex.Library) string { return lib.Latest.Website }, } +// matcherFromQueryString returns a closure that takes a library as a +// parameter and returns true if the library matches the query. +func matcherFromQueryString(query string) func(*librariesindex.Library) bool { + // A qv-query is one using [:=] syntax. + qvQuery := strings.Contains(query, ":") || strings.Contains(query, "=") + + if !qvQuery { + queryTerms := utils.SearchTermsFromQueryString(query) + return func(lib *librariesindex.Library) bool { + return utils.Match(defaultLibraryMatchExtractor(lib), queryTerms) + } + } + queryTerms := matcherTokensFromQueryString(query) return func(lib *librariesindex.Library) bool { @@ -134,15 +130,15 @@ var qualifiers map[string]func(lib *librariesindex.Library) string = map[string] // Flag indicating whether the search term matched a known qualifier knownQualifier := false - for _, q := range qualifiers { - if strings.HasPrefix(term, q.key+":") { - target := strings.TrimPrefix(term, q.key+":") - matched = (matched && utils.Match(q.extractor(lib), []string{target})) + for key, extractor := range qualifiers { + if strings.HasPrefix(term, key+":") { + target := strings.TrimPrefix(term, key+":") + matched = (matched && utils.Match(extractor(lib), []string{target})) knownQualifier = true break - } else if strings.HasPrefix(term, q.key+"=") { - target := strings.TrimPrefix(term, q.key+"=") - matched = (matched && strings.ToLower(q.extractor(lib)) == target) + } else if strings.HasPrefix(term, key+"=") { + target := strings.TrimPrefix(term, key+"=") + matched = (matched && strings.ToLower(extractor(lib)) == target) knownQualifier = true break } From eb4e1f1ab7d50385a133898d30ecbe626d05e576 Mon Sep 17 00:00:00 2001 From: Zachary Vonler Date: Fri, 20 Oct 2023 10:37:09 -0500 Subject: [PATCH 05/16] Eliminated loop over qualifiers. --- commands/lib/search.go | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/commands/lib/search.go b/commands/lib/search.go index da582dd738f..dda71d8663f 100644 --- a/commands/lib/search.go +++ b/commands/lib/search.go @@ -127,24 +127,24 @@ func matcherFromQueryString(query string) func(*librariesindex.Library) bool { matched := true for _, term := range queryTerms { - // Flag indicating whether the search term matched a known qualifier - knownQualifier := false - - for key, extractor := range qualifiers { - if strings.HasPrefix(term, key+":") { - target := strings.TrimPrefix(term, key+":") - matched = (matched && utils.Match(extractor(lib), []string{target})) - knownQualifier = true - break - } else if strings.HasPrefix(term, key+"=") { - target := strings.TrimPrefix(term, key+"=") - matched = (matched && strings.ToLower(extractor(lib)) == target) - knownQualifier = true - break + if sepIdx := strings.IndexAny(term, "=:"); sepIdx != -1 { + potentialKey := term[:sepIdx] + separator := term[sepIdx] + + extractor, ok := qualifiers[potentialKey] + if ok { + target := term[sepIdx+1:] + if separator == ':' { + matched = (matched && utils.Match(extractor(lib), []string{target})) + } else { // "=" + matched = (matched && strings.ToLower(extractor(lib)) == target) + } + } else { + // Unknown qualifier names revert to basic search terms. + matched = (matched && utils.Match(defaultLibraryMatchExtractor(lib), []string{term})) } - } - - if !knownQualifier { + } else { + // Terms that do not use qv-syntax are handled as usual. matched = (matched && utils.Match(defaultLibraryMatchExtractor(lib), []string{term})) } } From 71a040e2c908e7a1cb8eaae45f4fa63d17ee8ca2 Mon Sep 17 00:00:00 2001 From: Zachary Vonler Date: Fri, 20 Oct 2023 11:32:19 -0500 Subject: [PATCH 06/16] More tests. --- commands/lib/search_test.go | 47 +++++++++-- .../qualified_search/library_index.json | 83 +++++++++++++++++++ 2 files changed, 124 insertions(+), 6 deletions(-) create mode 100644 commands/lib/testdata/qualified_search/library_index.json diff --git a/commands/lib/search_test.go b/commands/lib/search_test.go index 109aa408c7c..e9003cb0946 100644 --- a/commands/lib/search_test.go +++ b/commands/lib/search_test.go @@ -28,6 +28,7 @@ import ( var customIndexPath = paths.New("testdata", "test1") var fullIndexPath = paths.New("testdata", "full") +var qualifiedSearchIndexPath = paths.New("testdata", "qualified_search") func TestSearchLibrary(t *testing.T) { lm := librariesmanager.NewLibraryManager(customIndexPath, nil) @@ -96,7 +97,7 @@ func TestSearchLibraryFields(t *testing.T) { } func TestSearchLibraryWithQualifiers(t *testing.T) { - lm := librariesmanager.NewLibraryManager(fullIndexPath, nil) + lm := librariesmanager.NewLibraryManager(qualifiedSearchIndexPath, nil) lm.LoadIndex() query := func(q string) []string { @@ -107,14 +108,48 @@ func TestSearchLibraryWithQualifiers(t *testing.T) { return libs } - res := query("name:FlashStorage") - require.Len(t, res, 7) + res := query("mesh") + require.Len(t, res, 4) - res = query("name=FlashStorage") - require.Len(t, res, 1) - require.Equal(t, "FlashStorage", res[0]) + res = query("name:Mesh") + require.Len(t, res, 3) + + res = query("name=Mesh") + require.Len(t, res, 0) + // Space not in double-quoted string + res = query("name=Painless Mesh") + require.Len(t, res, 0) + + // Embedded space in double-quoted string res = query("name=\"Painless Mesh\"") require.Len(t, res, 1) require.Equal(t, "Painless Mesh", res[0]) + + // No closing double-quote - still tokenizes with embedded space + res = query("name:\"Painless Mesh") + require.Len(t, res, 1) + + // Malformed double-quoted string with escaped first double-quote + res = query("name:\\\"Painless Mesh\"") + require.Len(t, res, 0) + + res = query("name:mesh author:TMRh20") + require.Len(t, res, 1) + require.Equal(t, "RF24Mesh", res[0]) + + res = query("mesh dependencies:ArduinoJson") + require.Len(t, res, 1) + require.Equal(t, "Painless Mesh", res[0]) + + res = query("architectures:esp author=\"Suraj I.\"") + require.Len(t, res, 1) + require.Equal(t, "esp8266-framework", res[0]) + + res = query("mesh esp") + require.Len(t, res, 2) + + res = query("mesh esp paragraph:wifi") + require.Len(t, res, 1) + require.Equal(t, "esp8266-framework", res[0]) } diff --git a/commands/lib/testdata/qualified_search/library_index.json b/commands/lib/testdata/qualified_search/library_index.json new file mode 100644 index 00000000000..c5e8d0e98d6 --- /dev/null +++ b/commands/lib/testdata/qualified_search/library_index.json @@ -0,0 +1,83 @@ +{ + "libraries": [ + { + "name": "esp8266-framework", + "version": "1.1.5", + "author": "Suraj I.", + "maintainer": "Suraj I. \u003csurajinamdar151@gmail.com\u003e", + "sentence": "esp8266 framework stack for easy configurable applications", + "paragraph": "esp8266 framework includes all services like gpio, wifi, http, mqtt, ntp, ota, napt, espnow, mesh, server etc. which are ready to use in all applications", + "website": "https://github.com/Suraj151/esp8266-framework", + "category": "Communication", + "architectures": ["esp8266"], + "types": ["Contributed"], + "repository": "https://github.com/Suraj151/esp8266-framework.git", + "url": "https://downloads.arduino.cc/libraries/github.com/Suraj151/esp8266_framework-1.1.5.zip", + "archiveFileName": "esp8266_framework-1.1.5.zip", + "size": 1918535, + "checksum": "SHA-256:81731d4ccc80846c317a2d4e2086d32caa695ed97d3e4765a59c5651b4be30b5" + }, + { + "name": "Painless Mesh", + "version": "1.5.0", + "author": "Coopdis,Scotty Franzyshen,Edwin van Leeuwen,Germán Martín,Maximilian Schwarz,Doanh Doanh", + "maintainer": "Edwin van Leeuwen", + "sentence": "A painless way to setup a mesh with ESP8266 and ESP32 devices", + "paragraph": "A painless way to setup a mesh with ESP8266 and ESP32 devices", + "website": "https://gitlab.com/painlessMesh/painlessMesh", + "category": "Communication", + "architectures": ["esp8266", "esp32"], + "types": ["Contributed"], + "repository": "https://gitlab.com/painlessMesh/painlessMesh.git", + "providesIncludes": ["painlessMesh.h"], + "dependencies": [ + { + "name": "ArduinoJson" + }, + { + "name": "TaskScheduler" + } + ], + "url": "https://downloads.arduino.cc/libraries/gitlab.com/painlessMesh/Painless_Mesh-1.5.0.zip", + "archiveFileName": "Painless_Mesh-1.5.0.zip", + "size": 293531, + "checksum": "SHA-256:9d965064fc704e8ba19c0452cc50e619145f7869b9b135dbf7e521f6ec0a4b33" + }, + { + "name": "RF24Mesh", + "version": "1.0.0", + "author": "TMRh20", + "maintainer": "TMRh20", + "sentence": "A library for NRF24L01(+) devices mesh.", + "paragraph": "Provides a simple and seamless 'mesh' layer for sensor networks, allowing automatic and dynamic configuration that can be customized to suit many scenarios. It is currently designed to interface directly with with the RF24Network Development library, an OSI Network Layer using nRF24L01(+) radios driven by the newly optimized RF24 library fork.", + "website": "http://tmrh20.github.io/RF24Mesh/", + "category": "Communication", + "architectures": ["avr"], + "types": ["Contributed"], + "repository": "https://github.com/TMRh20/RF24Mesh.git", + "url": "https://downloads.arduino.cc/libraries/github.com/TMRh20/RF24Mesh-1.0.0.zip", + "archiveFileName": "RF24Mesh-1.0.0.zip", + "size": 31419, + "checksum": "SHA-256:1b122a6412bc06a33a7fbcef34e2210d0990c25839fd7bc547604103f28194b5" + }, + { + "name": "DLLN3X ZigBee Mesh Module Library", + "version": "1.0.1", + "author": "Duke Liu \u003cmentalflow@ourdocs.cn\u003e", + "maintainer": "Duke Liu \u003cmentalflow@ourdocs.cn\u003e", + "license": "MIT", + "sentence": "This library allows you to use DLLN3X ZigBee mesh module very easily.", + "paragraph": "This library currently allows basic send and receive operations using the DLLN3X module, with more features to come.", + "website": "https://github.com/mentalfl0w/DLLN3X_zigbee_mesh_module_library", + "category": "Communication", + "architectures": ["*"], + "types": ["Contributed"], + "repository": "https://github.com/mentalfl0w/DLLN3X_zigbee_mesh_module_library.git", + "providesIncludes": ["DLLN3X.h"], + "url": "https://downloads.arduino.cc/libraries/github.com/mentalfl0w/DLLN3X_ZigBee_Mesh_Module_Library-1.0.1.zip", + "archiveFileName": "DLLN3X_ZigBee_Mesh_Module_Library-1.0.1.zip", + "size": 6122, + "checksum": "SHA-256:a28833bbd575ef8deab744a1f0e1175dad9e5329bf5c620fc2fe53e1de1d32ba" + } + ] +} From 47444d77f677de7b6c77182ede3c1b53fd939802 Mon Sep 17 00:00:00 2001 From: Zachary Vonler Date: Fri, 20 Oct 2023 13:42:40 -0500 Subject: [PATCH 07/16] Even more tests. --- commands/lib/search_test.go | 35 ++++++++++++++++++ .../qualified_search/library_index.json | 36 +++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/commands/lib/search_test.go b/commands/lib/search_test.go index e9003cb0946..133a8966e6c 100644 --- a/commands/lib/search_test.go +++ b/commands/lib/search_test.go @@ -152,4 +152,39 @@ func TestSearchLibraryWithQualifiers(t *testing.T) { res = query("mesh esp paragraph:wifi") require.Len(t, res, 1) require.Equal(t, "esp8266-framework", res[0]) + + // Unknown qualifier should revert to original matching + res = query("std::array") + require.Len(t, res, 1) + require.Equal(t, "Array", res[0]) + + res = query("data storage") + require.Len(t, res, 1) + require.Equal(t, "Pushdata_ESP8266_SSL", res[0]) + + res = query("category:\"data storage\"") + require.Len(t, res, 1) + require.Equal(t, "Array", res[0]) + + res = query("maintainer:@") + require.Len(t, res, 4) + + res = query("sentence:\"A library for NRF24L01(+) devices mesh.\"") + require.Len(t, res, 1) + require.Equal(t, "RF24Mesh", res[0]) + + res = query("types=contributed") + require.Len(t, res, 6) + + res = query("version:1.0") + require.Len(t, res, 2) + + res = query("version=1.2.1") + require.Len(t, res, 1) + require.Equal(t, "Array", res[0]) + + // Non-SSL URLs + res = query("website:http://") + require.Len(t, res, 1) + require.Equal(t, "RF24Mesh", res[0]) } diff --git a/commands/lib/testdata/qualified_search/library_index.json b/commands/lib/testdata/qualified_search/library_index.json index c5e8d0e98d6..0acfb4c3cfc 100644 --- a/commands/lib/testdata/qualified_search/library_index.json +++ b/commands/lib/testdata/qualified_search/library_index.json @@ -1,5 +1,22 @@ { "libraries": [ + { + "name": "Array", + "version": "1.2.1", + "author": "Peter Polidoro \u003cpeterpolidoro@gmail.com\u003e", + "maintainer": "Peter Polidoro \u003cpeterpolidoro@gmail.com\u003e", + "sentence": "An array container similar to the C++ std::array", + "paragraph": "Like this project? Please star it on GitHub!", + "website": "https://github.com/janelia-arduino/Array.git", + "category": "Data Storage", + "architectures": ["*"], + "types": ["Contributed"], + "repository": "https://github.com/janelia-arduino/Array.git", + "url": "https://downloads.arduino.cc/libraries/github.com/janelia-arduino/Array-1.2.1.zip", + "archiveFileName": "Array-1.2.1.zip", + "size": 7859, + "checksum": "SHA-256:dc69e0b4d1390c08253120a80e6e07e5cc6185ec24cbe3cb96dec2d8173e6495" + }, { "name": "esp8266-framework", "version": "1.1.5", @@ -43,6 +60,25 @@ "size": 293531, "checksum": "SHA-256:9d965064fc704e8ba19c0452cc50e619145f7869b9b135dbf7e521f6ec0a4b33" }, + { + "name": "Pushdata_ESP8266_SSL", + "version": "0.0.6", + "author": "Ragnar Lonn", + "maintainer": "Ragnar Lonn \u003chello@pushdata.io\u003e", + "license": "MIT", + "sentence": "Free, ultra-simple time series data storage for your IoT sensors", + "paragraph": "Pushdata.io client library that makes it very simple to store your time series data online", + "website": "https://pushdata.io", + "category": "Communication", + "architectures": ["*"], + "types": ["Contributed"], + "repository": "https://github.com/pushdata-io/Arduino_ESP8266_SSL.git", + "providesIncludes": ["Pushdata_ESP8266_SSL.h"], + "url": "https://downloads.arduino.cc/libraries/github.com/pushdata-io/Pushdata_ESP8266_SSL-0.0.6.zip", + "archiveFileName": "Pushdata_ESP8266_SSL-0.0.6.zip", + "size": 12160, + "checksum": "SHA-256:5d592eb7900782f681b86f5fd77c5d9f25c78555e3b5f0880c52197031206df0" + }, { "name": "RF24Mesh", "version": "1.0.0", From 4e04a2710ad4d811a5765219a769d9a824161bb8 Mon Sep 17 00:00:00 2001 From: Zachary Vonler Date: Fri, 20 Oct 2023 13:52:14 -0500 Subject: [PATCH 08/16] Moved MatcherTokensFromQueryString and supporting logic to their own file. --- commands/lib/search.go | 117 +-------------------------- commands/lib/search_matcher.go | 139 +++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 116 deletions(-) create mode 100644 commands/lib/search_matcher.go diff --git a/commands/lib/search.go b/commands/lib/search.go index dda71d8663f..91a8ddaabcb 100644 --- a/commands/lib/search.go +++ b/commands/lib/search.go @@ -23,7 +23,6 @@ import ( "github.com/arduino/arduino-cli/arduino" "github.com/arduino/arduino-cli/arduino/libraries/librariesindex" "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" - "github.com/arduino/arduino-cli/arduino/utils" "github.com/arduino/arduino-cli/commands/internal/instances" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" semver "go.bug.st/relaxed-semver" @@ -38,120 +37,6 @@ func LibrarySearch(ctx context.Context, req *rpc.LibrarySearchRequest) (*rpc.Lib return searchLibrary(req, lm), nil } -// matcherTokensFromQueryString parses the query string into tokens of interest -// for the qualifier-value pattern matching. -func matcherTokensFromQueryString(query string) []string { - escaped := false - quoted := false - tokens := []string{} - sb := &strings.Builder{} - - for _, r := range query { - // Short circuit the loop on backslash so that all other paths can clear - // the escaped flag. - if !escaped && r == '\\' { - escaped = true - continue - } - - if r == '"' { - if !escaped { - quoted = !quoted - } else { - sb.WriteRune(r) - } - } else if !quoted && r == ' ' { - tokens = append(tokens, strings.ToLower(sb.String())) - sb.Reset() - } else { - sb.WriteRune(r) - } - escaped = false - } - if sb.Len() > 0 { - tokens = append(tokens, strings.ToLower(sb.String())) - } - - return tokens -} - -// defaulLibraryMatchExtractor returns a string describing the library that -// is used for the simple search. -func defaultLibraryMatchExtractor(lib *librariesindex.Library) string { - res := lib.Name + " " + - lib.Latest.Paragraph + " " + - lib.Latest.Sentence + " " + - lib.Latest.Author + " " - for _, include := range lib.Latest.ProvidesIncludes { - res += include + " " - } - return res -} - -var qualifiers map[string]func(lib *librariesindex.Library) string = map[string]func(lib *librariesindex.Library) string{ - "name": func(lib *librariesindex.Library) string { return lib.Name }, - "architectures": func(lib *librariesindex.Library) string { return strings.Join(lib.Latest.Architectures, " ") }, - "author": func(lib *librariesindex.Library) string { return lib.Latest.Author }, - "category": func(lib *librariesindex.Library) string { return lib.Latest.Category }, - "dependencies": func(lib *librariesindex.Library) string { - names := make([]string, len(lib.Latest.Dependencies)) - for i, dep := range lib.Latest.Dependencies { - names[i] = dep.GetName() - } - return strings.Join(names, " ") - }, - "maintainer": func(lib *librariesindex.Library) string { return lib.Latest.Maintainer }, - "paragraph": func(lib *librariesindex.Library) string { return lib.Latest.Paragraph }, - "sentence": func(lib *librariesindex.Library) string { return lib.Latest.Sentence }, - "types": func(lib *librariesindex.Library) string { return strings.Join(lib.Latest.Types, " ") }, - "version": func(lib *librariesindex.Library) string { return lib.Latest.Version.String() }, - "website": func(lib *librariesindex.Library) string { return lib.Latest.Website }, -} - -// matcherFromQueryString returns a closure that takes a library as a -// parameter and returns true if the library matches the query. -func matcherFromQueryString(query string) func(*librariesindex.Library) bool { - // A qv-query is one using [:=] syntax. - qvQuery := strings.Contains(query, ":") || strings.Contains(query, "=") - - if !qvQuery { - queryTerms := utils.SearchTermsFromQueryString(query) - return func(lib *librariesindex.Library) bool { - return utils.Match(defaultLibraryMatchExtractor(lib), queryTerms) - } - } - - queryTerms := matcherTokensFromQueryString(query) - - return func(lib *librariesindex.Library) bool { - matched := true - for _, term := range queryTerms { - - if sepIdx := strings.IndexAny(term, "=:"); sepIdx != -1 { - potentialKey := term[:sepIdx] - separator := term[sepIdx] - - extractor, ok := qualifiers[potentialKey] - if ok { - target := term[sepIdx+1:] - if separator == ':' { - matched = (matched && utils.Match(extractor(lib), []string{target})) - } else { // "=" - matched = (matched && strings.ToLower(extractor(lib)) == target) - } - } else { - // Unknown qualifier names revert to basic search terms. - matched = (matched && utils.Match(defaultLibraryMatchExtractor(lib), []string{term})) - } - } else { - // Terms that do not use qv-syntax are handled as usual. - matched = (matched && utils.Match(defaultLibraryMatchExtractor(lib), []string{term})) - } - } - return matched - } -} - func searchLibrary(req *rpc.LibrarySearchRequest, lm *librariesmanager.LibrariesManager) *rpc.LibrarySearchResponse { res := []*rpc.SearchedLibrary{} query := req.GetSearchArgs() @@ -159,7 +44,7 @@ func searchLibrary(req *rpc.LibrarySearchRequest, lm *librariesmanager.Libraries query = req.GetQuery() } - matcher := matcherFromQueryString(query) + matcher := MatcherFromQueryString(query) for _, lib := range lm.Index.Libraries { if matcher(lib) { diff --git a/commands/lib/search_matcher.go b/commands/lib/search_matcher.go new file mode 100644 index 00000000000..402fb3af024 --- /dev/null +++ b/commands/lib/search_matcher.go @@ -0,0 +1,139 @@ +// This file is part of arduino-cli. +// +// Copyright 2023 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package lib + +import ( + "strings" + + "github.com/arduino/arduino-cli/arduino/libraries/librariesindex" + "github.com/arduino/arduino-cli/arduino/utils" +) + +// matcherTokensFromQueryString parses the query string into tokens of interest +// for the qualifier-value pattern matching. +func matcherTokensFromQueryString(query string) []string { + escaped := false + quoted := false + tokens := []string{} + sb := &strings.Builder{} + + for _, r := range query { + // Short circuit the loop on backslash so that all other paths can clear + // the escaped flag. + if !escaped && r == '\\' { + escaped = true + continue + } + + if r == '"' { + if !escaped { + quoted = !quoted + } else { + sb.WriteRune(r) + } + } else if !quoted && r == ' ' { + tokens = append(tokens, strings.ToLower(sb.String())) + sb.Reset() + } else { + sb.WriteRune(r) + } + escaped = false + } + if sb.Len() > 0 { + tokens = append(tokens, strings.ToLower(sb.String())) + } + + return tokens +} + +// defaulLibraryMatchExtractor returns a string describing the library that +// is used for the simple search. +func defaultLibraryMatchExtractor(lib *librariesindex.Library) string { + res := lib.Name + " " + + lib.Latest.Paragraph + " " + + lib.Latest.Sentence + " " + + lib.Latest.Author + " " + for _, include := range lib.Latest.ProvidesIncludes { + res += include + " " + } + return res +} + +var qualifiers map[string]func(lib *librariesindex.Library) string = map[string]func(lib *librariesindex.Library) string{ + "name": func(lib *librariesindex.Library) string { return lib.Name }, + "architectures": func(lib *librariesindex.Library) string { return strings.Join(lib.Latest.Architectures, " ") }, + "author": func(lib *librariesindex.Library) string { return lib.Latest.Author }, + "category": func(lib *librariesindex.Library) string { return lib.Latest.Category }, + "dependencies": func(lib *librariesindex.Library) string { + names := make([]string, len(lib.Latest.Dependencies)) + for i, dep := range lib.Latest.Dependencies { + names[i] = dep.GetName() + } + return strings.Join(names, " ") + }, + "maintainer": func(lib *librariesindex.Library) string { return lib.Latest.Maintainer }, + "paragraph": func(lib *librariesindex.Library) string { return lib.Latest.Paragraph }, + "sentence": func(lib *librariesindex.Library) string { return lib.Latest.Sentence }, + "types": func(lib *librariesindex.Library) string { return strings.Join(lib.Latest.Types, " ") }, + "version": func(lib *librariesindex.Library) string { return lib.Latest.Version.String() }, + "website": func(lib *librariesindex.Library) string { return lib.Latest.Website }, +} + +// matcherFromQueryString returns a closure that takes a library as a +// parameter and returns true if the library matches the query. +func MatcherFromQueryString(query string) func(*librariesindex.Library) bool { + // A qv-query is one using [:=] syntax. + qvQuery := strings.Contains(query, ":") || strings.Contains(query, "=") + + if !qvQuery { + queryTerms := utils.SearchTermsFromQueryString(query) + return func(lib *librariesindex.Library) bool { + return utils.Match(defaultLibraryMatchExtractor(lib), queryTerms) + } + } + + queryTerms := matcherTokensFromQueryString(query) + + return func(lib *librariesindex.Library) bool { + matched := true + for _, term := range queryTerms { + + if sepIdx := strings.IndexAny(term, "=:"); sepIdx != -1 { + potentialKey := term[:sepIdx] + separator := term[sepIdx] + + extractor, ok := qualifiers[potentialKey] + if ok { + target := term[sepIdx+1:] + if separator == ':' { + matched = (matched && utils.Match(extractor(lib), []string{target})) + } else { // "=" + matched = (matched && strings.ToLower(extractor(lib)) == target) + } + } else { + // Unknown qualifier names revert to basic search terms. + matched = (matched && utils.Match(defaultLibraryMatchExtractor(lib), []string{term})) + } + } else { + // Terms that do not use qv-syntax are handled as usual. + matched = (matched && utils.Match(defaultLibraryMatchExtractor(lib), []string{term})) + } + } + return matched + } +} + + From 2f8baf24d96ecbafbeab226b2f4559bc4ea205dd Mon Sep 17 00:00:00 2001 From: Zachary Vonler Date: Fri, 20 Oct 2023 14:03:06 -0500 Subject: [PATCH 09/16] whitespace --- commands/lib/search_matcher.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/commands/lib/search_matcher.go b/commands/lib/search_matcher.go index 402fb3af024..c6b21e644e4 100644 --- a/commands/lib/search_matcher.go +++ b/commands/lib/search_matcher.go @@ -135,5 +135,3 @@ func MatcherFromQueryString(query string) func(*librariesindex.Library) bool { return matched } } - - From e741a3211350d6315c37c1b0484f598be51b6942 Mon Sep 17 00:00:00 2001 From: Zachary Vonler Date: Fri, 20 Oct 2023 14:26:33 -0500 Subject: [PATCH 10/16] Added integration test. --- commands/lib/search_test.go | 9 ++++- .../qualified_search/library_index.json | 17 +++++++++ internal/integrationtest/lib/lib_test.go | 37 +++++++++++++++++++ 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/commands/lib/search_test.go b/commands/lib/search_test.go index 133a8966e6c..aa63952e1bd 100644 --- a/commands/lib/search_test.go +++ b/commands/lib/search_test.go @@ -174,10 +174,10 @@ func TestSearchLibraryWithQualifiers(t *testing.T) { require.Equal(t, "RF24Mesh", res[0]) res = query("types=contributed") - require.Len(t, res, 6) + require.Len(t, res, 7) res = query("version:1.0") - require.Len(t, res, 2) + require.Len(t, res, 3) res = query("version=1.2.1") require.Len(t, res, 1) @@ -187,4 +187,9 @@ func TestSearchLibraryWithQualifiers(t *testing.T) { res = query("website:http://") require.Len(t, res, 1) require.Equal(t, "RF24Mesh", res[0]) + + // Literal double-quote + res = query("sentence:\\\"") + require.Len(t, res, 1) + require.Equal(t, "RTCtime", res[0]) } diff --git a/commands/lib/testdata/qualified_search/library_index.json b/commands/lib/testdata/qualified_search/library_index.json index 0acfb4c3cfc..23dfb04bae9 100644 --- a/commands/lib/testdata/qualified_search/library_index.json +++ b/commands/lib/testdata/qualified_search/library_index.json @@ -96,6 +96,23 @@ "size": 31419, "checksum": "SHA-256:1b122a6412bc06a33a7fbcef34e2210d0990c25839fd7bc547604103f28194b5" }, + { + "name": "RTCtime", + "version": "1.0.5", + "author": "smz \u003ctinker@smz.it\u003e", + "maintainer": "smz (https://github.com/smz)", + "sentence": "A \"Standard C Runtime\" compatible library for interfacing the DS1307 and DS3231 Real Time Clock modules.", + "paragraph": "This library is for getting/setting time from hardware RTC modules. It uses an API compatible with the AVR implementation of the Standard C runtime time library as available in the Arduino IDE since version 1.6.10 (AVR C Runtime Library 2.0.0)", + "website": "https://github.com/smz/Arduino-RTCtime", + "category": "Timing", + "architectures": ["*"], + "types": ["Contributed"], + "repository": "https://github.com/smz/Arduino-RTCtime.git", + "url": "https://downloads.arduino.cc/libraries/github.com/smz/RTCtime-1.0.5.zip", + "archiveFileName": "RTCtime-1.0.5.zip", + "size": 18870, + "checksum": "SHA-256:89493bb6d1f834426e82330fdf55a249ff43eb61707831d75deed8644a7ebce8" + }, { "name": "DLLN3X ZigBee Mesh Module Library", "version": "1.0.1", diff --git a/internal/integrationtest/lib/lib_test.go b/internal/integrationtest/lib/lib_test.go index f064f32e741..9e6c8f8a040 100644 --- a/internal/integrationtest/lib/lib_test.go +++ b/internal/integrationtest/lib/lib_test.go @@ -788,6 +788,43 @@ func TestSearch(t *testing.T) { runSearch("json", []string{"ArduinoJson", "Arduino_JSON"}) } +func TestQualifiedSearch(t *testing.T) { + env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t) + defer env.CleanUp() + + stdout, _, err := cli.Run("lib", "search", "--names", "WiFi101") + require.NoError(t, err) + lines := strings.Split(strings.TrimSpace(string(stdout)), "\n") + var libs []string + for i, v := range lines { + lines[i] = strings.TrimSpace(v) + if strings.Contains(v, "Name:") { + libs = append(libs, strings.Trim(strings.SplitN(v, " ", 2)[1], "\"")) + } + } + + expected := []string{"WiFi101", "WiFi101OTA", "Firebase Arduino based on WiFi101", "WiFi101_Generic"} + require.Subset(t, libs, expected) + + runSearch := func(args string, expectedLibs []string) { + stdout, _, err = cli.Run("lib", "search", "--names", "--json", args) + require.NoError(t, err) + libraries := requirejson.Parse(t, stdout).Query("[ .libraries | .[] | .name ]").String() + for _, l := range expectedLibs { + require.Contains(t, libraries, l) + } + } + runSearch("name:MKRIoTCarrier", []string{"Arduino_MKRIoTCarrier"}) + runSearch("name=Arduino_MKRIoTCarrier", []string{"Arduino_MKRIoTCarrier"}) + // Embedded space in double-quoted string + runSearch("name=\"dht sensor library\"", []string{"DHT sensor library", "DHT sensor library for ESPx", "SimpleDHT", "SDHT"}) + // No closing double-quote + runSearch("name=\"dht sensor library", []string{"DHT sensor library", "DHT sensor library for ESPx", "SimpleDHT", "SDHT"}) + runSearch("name:\"sensor dht\"", []string{}) + // Literal double-quote + runSearch("sentence:\\\"", []string{"RTCtime"}) +} + func TestSearchParagraph(t *testing.T) { env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t) defer env.CleanUp() From 933cc5bb84799ddbe9e0ebdbb00272fd981b9985 Mon Sep 17 00:00:00 2001 From: Zachary Vonler Date: Fri, 20 Oct 2023 14:38:23 -0500 Subject: [PATCH 11/16] Fixed options to switch. --- internal/integrationtest/lib/lib_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/integrationtest/lib/lib_test.go b/internal/integrationtest/lib/lib_test.go index 9e6c8f8a040..2f8c0de61b4 100644 --- a/internal/integrationtest/lib/lib_test.go +++ b/internal/integrationtest/lib/lib_test.go @@ -807,7 +807,7 @@ func TestQualifiedSearch(t *testing.T) { require.Subset(t, libs, expected) runSearch := func(args string, expectedLibs []string) { - stdout, _, err = cli.Run("lib", "search", "--names", "--json", args) + stdout, _, err = cli.Run("lib", "search", "--names", "--format", "json", args) require.NoError(t, err) libraries := requirejson.Parse(t, stdout).Query("[ .libraries | .[] | .name ]").String() for _, l := range expectedLibs { From 15b07a5d711602fc6198f473a5566af22d1243de Mon Sep 17 00:00:00 2001 From: Zachary Vonler Date: Fri, 20 Oct 2023 14:48:10 -0500 Subject: [PATCH 12/16] Fixed test expectations. --- internal/integrationtest/lib/lib_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/integrationtest/lib/lib_test.go b/internal/integrationtest/lib/lib_test.go index 2f8c0de61b4..48597345272 100644 --- a/internal/integrationtest/lib/lib_test.go +++ b/internal/integrationtest/lib/lib_test.go @@ -817,9 +817,9 @@ func TestQualifiedSearch(t *testing.T) { runSearch("name:MKRIoTCarrier", []string{"Arduino_MKRIoTCarrier"}) runSearch("name=Arduino_MKRIoTCarrier", []string{"Arduino_MKRIoTCarrier"}) // Embedded space in double-quoted string - runSearch("name=\"dht sensor library\"", []string{"DHT sensor library", "DHT sensor library for ESPx", "SimpleDHT", "SDHT"}) + runSearch("name=\"dht sensor library\"", []string{"DHT sensor library"}) // No closing double-quote - runSearch("name=\"dht sensor library", []string{"DHT sensor library", "DHT sensor library for ESPx", "SimpleDHT", "SDHT"}) + runSearch("name=\"dht sensor library", []string{"DHT sensor library"}) runSearch("name:\"sensor dht\"", []string{}) // Literal double-quote runSearch("sentence:\\\"", []string{"RTCtime"}) From f9208f8fa516830660d960744b5b8a254c975dd8 Mon Sep 17 00:00:00 2001 From: Zachary Vonler Date: Fri, 20 Oct 2023 15:05:42 -0500 Subject: [PATCH 13/16] Capitalized --- commands/lib/search_matcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/lib/search_matcher.go b/commands/lib/search_matcher.go index c6b21e644e4..69c1b776802 100644 --- a/commands/lib/search_matcher.go +++ b/commands/lib/search_matcher.go @@ -92,7 +92,7 @@ var qualifiers map[string]func(lib *librariesindex.Library) string = map[string] "website": func(lib *librariesindex.Library) string { return lib.Latest.Website }, } -// matcherFromQueryString returns a closure that takes a library as a +// MatcherFromQueryString returns a closure that takes a library as a // parameter and returns true if the library matches the query. func MatcherFromQueryString(query string) func(*librariesindex.Library) bool { // A qv-query is one using [:=] syntax. From 192488180178751e671999765a5ff16516abb281 Mon Sep 17 00:00:00 2001 From: Zachary Vonler Date: Fri, 20 Oct 2023 15:09:50 -0500 Subject: [PATCH 14/16] Removed redundant test copy/paste. --- internal/integrationtest/lib/lib_test.go | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/internal/integrationtest/lib/lib_test.go b/internal/integrationtest/lib/lib_test.go index 48597345272..b00948a0c4c 100644 --- a/internal/integrationtest/lib/lib_test.go +++ b/internal/integrationtest/lib/lib_test.go @@ -792,22 +792,8 @@ func TestQualifiedSearch(t *testing.T) { env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t) defer env.CleanUp() - stdout, _, err := cli.Run("lib", "search", "--names", "WiFi101") - require.NoError(t, err) - lines := strings.Split(strings.TrimSpace(string(stdout)), "\n") - var libs []string - for i, v := range lines { - lines[i] = strings.TrimSpace(v) - if strings.Contains(v, "Name:") { - libs = append(libs, strings.Trim(strings.SplitN(v, " ", 2)[1], "\"")) - } - } - - expected := []string{"WiFi101", "WiFi101OTA", "Firebase Arduino based on WiFi101", "WiFi101_Generic"} - require.Subset(t, libs, expected) - runSearch := func(args string, expectedLibs []string) { - stdout, _, err = cli.Run("lib", "search", "--names", "--format", "json", args) + stdout, _, err := cli.Run("lib", "search", "--names", "--format", "json", args) require.NoError(t, err) libraries := requirejson.Parse(t, stdout).Query("[ .libraries | .[] | .name ]").String() for _, l := range expectedLibs { From 9c211a99644989aa2e1e4bef5d5fe97da5b54975 Mon Sep 17 00:00:00 2001 From: Zach Vonler Date: Fri, 20 Oct 2023 18:09:51 -0500 Subject: [PATCH 15/16] Update commands/lib/search_matcher.go Co-authored-by: Alessio Perugini --- commands/lib/search_matcher.go | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/commands/lib/search_matcher.go b/commands/lib/search_matcher.go index 69c1b776802..df856536c53 100644 --- a/commands/lib/search_matcher.go +++ b/commands/lib/search_matcher.go @@ -110,27 +110,23 @@ func MatcherFromQueryString(query string) func(*librariesindex.Library) bool { return func(lib *librariesindex.Library) bool { matched := true for _, term := range queryTerms { - - if sepIdx := strings.IndexAny(term, "=:"); sepIdx != -1 { - potentialKey := term[:sepIdx] - separator := term[sepIdx] - - extractor, ok := qualifiers[potentialKey] - if ok { - target := term[sepIdx+1:] - if separator == ':' { + if sepIdx := strings.IndexAny(term, ":="); sepIdx != -1 { + qualifier, separator, target := term[:sepIdx], term[sepIdx], term[sepIdx+1:] + if extractor, ok := qualifiers[qualifier]; ok { + switch separator { + case ':': matched = (matched && utils.Match(extractor(lib), []string{target})) - } else { // "=" + continue + case '=': matched = (matched && strings.ToLower(extractor(lib)) == target) + continue } - } else { - // Unknown qualifier names revert to basic search terms. - matched = (matched && utils.Match(defaultLibraryMatchExtractor(lib), []string{term})) } - } else { - // Terms that do not use qv-syntax are handled as usual. - matched = (matched && utils.Match(defaultLibraryMatchExtractor(lib), []string{term})) } + // We perform the usual match in the following cases: + // 1. Unknown qualifier names revert to basic search terms. + // 2. Terms that do not use qv-syntax. + matched = (matched && utils.Match(defaultLibraryMatchExtractor(lib), []string{term})) } return matched } From 9fab459f36a96813340bfe06328c6603b634bdfc Mon Sep 17 00:00:00 2001 From: Zachary Vonler Date: Mon, 23 Oct 2023 10:56:41 -0500 Subject: [PATCH 16/16] Added QV syntax documentation. --- commands/lib/search_matcher.go | 2 ++ commands/lib/search_test.go | 11 +++++++ internal/cli/lib/search.go | 59 +++++++++++++++++++++++++++++++--- 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/commands/lib/search_matcher.go b/commands/lib/search_matcher.go index df856536c53..193246579ca 100644 --- a/commands/lib/search_matcher.go +++ b/commands/lib/search_matcher.go @@ -84,8 +84,10 @@ var qualifiers map[string]func(lib *librariesindex.Library) string = map[string] } return strings.Join(names, " ") }, + "license": func(lib *librariesindex.Library) string { return lib.Latest.License }, "maintainer": func(lib *librariesindex.Library) string { return lib.Latest.Maintainer }, "paragraph": func(lib *librariesindex.Library) string { return lib.Latest.Paragraph }, + "provides": func(lib *librariesindex.Library) string { return strings.Join(lib.Latest.ProvidesIncludes, " ") }, "sentence": func(lib *librariesindex.Library) string { return lib.Latest.Sentence }, "types": func(lib *librariesindex.Library) string { return strings.Join(lib.Latest.Types, " ") }, "version": func(lib *librariesindex.Library) string { return lib.Latest.Version.String() }, diff --git a/commands/lib/search_test.go b/commands/lib/search_test.go index aa63952e1bd..8a5870338cd 100644 --- a/commands/lib/search_test.go +++ b/commands/lib/search_test.go @@ -192,4 +192,15 @@ func TestSearchLibraryWithQualifiers(t *testing.T) { res = query("sentence:\\\"") require.Len(t, res, 1) require.Equal(t, "RTCtime", res[0]) + + res = query("license=MIT") + require.Len(t, res, 2) + + // Empty string + res = query("license=\"\"") + require.Len(t, res, 5) + + res = query("provides:painlessmesh.h") + require.Len(t, res, 1) + require.Equal(t, "Painless Mesh", res[0]) } diff --git a/internal/cli/lib/search.go b/internal/cli/lib/search.go index c44ced3778b..9af3c7c1079 100644 --- a/internal/cli/lib/search.go +++ b/internal/cli/lib/search.go @@ -37,11 +37,60 @@ func initSearchCommand() *cobra.Command { var namesOnly bool var omitReleasesDetails bool searchCommand := &cobra.Command{ - Use: fmt.Sprintf("search [%s]", tr("LIBRARY_NAME")), - Short: tr("Searches for one or more libraries data."), - Long: tr("Search for one or more libraries data (case insensitive search)."), - Example: " " + os.Args[0] + " lib search audio", - Args: cobra.ArbitraryArgs, + Use: fmt.Sprintf("search [%s ...]", tr("SEARCH_TERM")), + Short: tr("Searches for one or more libraries matching a query."), + Long: tr(`Search for libraries matching zero or more search terms. + +All searches are performed in a case-insensitive fashion. Queries containing +multiple search terms will return only libraries that match all of the terms. + +Search terms that do not match the QV syntax described below are basic search +terms, and will match libraries that include the term anywhere in any of the +following fields: + - Author + - Name + - Paragraph + - Provides + - Sentence + +A special syntax, called qualifier-value (QV), indicates that a search term +should be compared against only one field of each library index entry. This +syntax uses the name of an index field (case-insensitive), an equals sign (=) +or a colon (:), and a value, e.g. 'name=ArduinoJson' or 'provides:tinyusb.h'. + +QV search terms that use a colon separator will match all libraries with the +value anywhere in the named field, and QV search terms that use an equals +separator will match only libraries with exactly the provided value in the +named field. + +QV search terms can include embedded spaces using double-quote (") characters +around the value or the entire term, e.g. 'category="Data Processing"' and +'"category=Data Processing"' are equivalent. A QV term can include a literal +double-quote character by preceding it with a backslash (\) character. + +NOTE: QV search terms using double-quote or backslash characters that are +passed as command-line arguments may require quoting or escaping to prevent +the shell from interpreting those characters. + +In addition to the fields listed above, QV terms can use these qualifiers: + - Architectures + - Category + - Dependencies + - License + - Maintainer + - Types + - Version + - Website + `), + Example: " " + os.Args[0] + " lib search audio # " + tr("basic search for \"audio\"") + "\n" + + " " + os.Args[0] + " lib search name:buzzer # " + tr("libraries with \"buzzer\" in the Name field") + "\n" + + " " + os.Args[0] + " lib search name=pcf8523 # " + tr("libraries with a Name exactly matching \"pcf8523\"") + "\n" + + " " + os.Args[0] + " lib search \"author:\\\"Daniel Garcia\\\"\" # " + tr("libraries authored by Daniel Garcia") + "\n" + + " " + os.Args[0] + " lib search author=Adafruit name:gfx # " + tr("libraries authored only by Adafruit with \"gfx\" in their Name") + "\n" + + " " + os.Args[0] + " lib search esp32 display maintainer=espressif # " + tr("basic search for \"esp32\" and \"display\" limited to official Maintainer") + "\n" + + " " + os.Args[0] + " lib search dependencies:IRremote # " + tr("libraries that depend on at least \"IRremote\"") + "\n" + + " " + os.Args[0] + " lib search dependencies=IRremote # " + tr("libraries that depend only on \"IRremote\"") + "\n", + Args: cobra.ArbitraryArgs, Run: func(cmd *cobra.Command, args []string) { runSearchCommand(args, namesOnly, omitReleasesDetails) },