diff --git a/src/config/defaults_ranking.go b/src/config/defaults_ranking.go index d40d2437..40a3b7cc 100644 --- a/src/config/defaults_ranking.go +++ b/src/config/defaults_ranking.go @@ -19,7 +19,7 @@ func EmptyRanking(engs []engines.Name) CategoryRanking { } for _, eng := range engs { - rnk.Engines[eng.ToLower()] = CategoryEngineRanking{ + rnk.Engines[eng.String()] = CategoryEngineRanking{ Mul: 1, Const: 0, } diff --git a/src/router/routes/responses.go b/src/router/routes/responses.go index 850a6efa..bd4dcbd2 100644 --- a/src/router/routes/responses.go +++ b/src/router/routes/responses.go @@ -13,10 +13,21 @@ type ErrorResponse struct { Value string `json:"value"` } +type responseBase struct { + Version string `json:"version"` + Duration int64 `json:"duration"` +} + type ResultsResponse struct { - Version string `json:"version"` - Duration int64 `json:"duration"` - Results []result.ResultOutput `json:"results"` + responseBase + + Results []result.ResultOutput `json:"results"` +} + +type SuggestionsResponse struct { + responseBase + + Suggestions []result.Suggestion `json:"suggestions"` } func writeResponse(w http.ResponseWriter, status int, body string) error { @@ -42,7 +53,7 @@ func writeResponseJSON(w http.ResponseWriter, status int, body any) error { return err } -func writeResponseSuggestions(w http.ResponseWriter, status int, query string, suggestions []string, api bool) error { +func writeResponseSuggestions(w http.ResponseWriter, status int, query string, suggestions []string) error { jsonStruct := make([]any, 2) jsonStruct[0] = query jsonStruct[1] = suggestions @@ -57,11 +68,7 @@ func writeResponseSuggestions(w http.ResponseWriter, status int, query string, s return err } - if api { - w.Header().Set("Content-Type", "application/x-suggestions+json") - } else { - w.Header().Set("Content-Type", "application/json") - } + w.Header().Set("Content-Type", "application/x-suggestions+json") w.WriteHeader(status) _, err = w.Write(res) return err diff --git a/src/router/routes/route_search.go b/src/router/routes/route_search.go index eb432180..d6a80f7a 100644 --- a/src/router/routes/route_search.go +++ b/src/router/routes/route_search.go @@ -3,6 +3,7 @@ package routes import ( "fmt" "net/http" + "slices" "strconv" "strings" "time" @@ -168,7 +169,8 @@ func routeSearch(w http.ResponseWriter, r *http.Request, ver string, catsConf ma } // Rank the results. - rankedRes := rank.Rank(scrapedRes, catsConf[categoryName].Ranking) + var rankedRes rank.Results = slices.Clone(scrapedRes) + rankedRes.Rank(catsConf[categoryName].Ranking) // Store the results in cache. if err := db.SetResults(query, categoryName, opts, rankedRes, ttlConf.Time); err != nil { @@ -184,9 +186,11 @@ func routeSearch(w http.ResponseWriter, r *http.Request, ver string, catsConf ma // Create the response. res := ResultsResponse{ - Version: ver, - Duration: time.Since(startTime).Milliseconds(), - Results: outpusRes, + responseBase{ + ver, + time.Since(startTime).Milliseconds(), + }, + outpusRes, } // If writing response failes, return the error. diff --git a/src/router/routes/route_suggest.go b/src/router/routes/route_suggest.go index b3061b1a..12b2da8e 100644 --- a/src/router/routes/route_suggest.go +++ b/src/router/routes/route_suggest.go @@ -3,15 +3,21 @@ package routes import ( "fmt" "net/http" + "slices" "strings" + "time" "github.com/hearchco/agent/src/config" "github.com/hearchco/agent/src/search" "github.com/hearchco/agent/src/search/engines/options" "github.com/hearchco/agent/src/search/result" + "github.com/hearchco/agent/src/search/result/rank" ) -func routeSuggest(w http.ResponseWriter, r *http.Request, catConf config.Category) error { +func routeSuggest(w http.ResponseWriter, r *http.Request, ver string, catConf config.Category) error { + // Capture start time. + startTime := time.Now() + // Parse form data (including query params). if err := r.ParseForm(); err != nil { // Server error. @@ -59,15 +65,28 @@ func routeSuggest(w http.ResponseWriter, r *http.Request, catConf config.Categor return err } - // TODO: Rank the suggestions. - // rankedSugs := rank.Rank(scrapedSugs, Ranking) - - // Convert the suggestions to output format. - outputSugs := result.ConvertSuggestionsToOutput(scrapedSugs) + // Rank the suggestions. + var rankedSugs rank.Suggestions = slices.Clone(scrapedSugs) + rankedSugs.Rank(catConf.Ranking) // Check if the response should be in API format or normal JSON format. - api := strings.Contains(r.Header.Get("Accept"), "application/x-suggestions+json") + if strings.Contains(r.Header.Get("Accept"), "application/x-suggestions+json") { + // Convert the suggestions to slice of strings. + stringSugs := result.ConvertSuggestionsToOutput(rankedSugs) - // If writing response failes, return the error. - return writeResponseSuggestions(w, http.StatusOK, query, outputSugs, api) + // If writing response failes, return the error. + return writeResponseSuggestions(w, http.StatusOK, query, stringSugs) + } else { + // Create the response. + res := SuggestionsResponse{ + responseBase{ + ver, + time.Since(startTime).Milliseconds(), + }, + rankedSugs, + } + + // If writing response failes, return the error. + return writeResponseJSON(w, http.StatusOK, res) + } } diff --git a/src/router/routes/setup.go b/src/router/routes/setup.go index 1d7a968c..eaf6cbc1 100644 --- a/src/router/routes/setup.go +++ b/src/router/routes/setup.go @@ -60,7 +60,7 @@ func Setup(mux *chi.Mux, ver string, db cache.DB, conf config.Config) { // /suggestions mux.Get("/suggestions", func(w http.ResponseWriter, r *http.Request) { - err := routeSuggest(w, r, conf.Categories[category.SUGGESTIONS]) + err := routeSuggest(w, r, ver, conf.Categories[category.SUGGESTIONS]) if err != nil { log.Error(). Err(err). @@ -70,7 +70,7 @@ func Setup(mux *chi.Mux, ver string, db cache.DB, conf config.Config) { } }) mux.Post("/suggestions", func(w http.ResponseWriter, r *http.Request) { - err := routeSuggest(w, r, conf.Categories[category.SUGGESTIONS]) + err := routeSuggest(w, r, ver, conf.Categories[category.SUGGESTIONS]) if err != nil { log.Error(). Err(err). diff --git a/src/search/engines/duckduckgo/suggest.go b/src/search/engines/duckduckgo/suggest.go index 1caf1bb3..a909285f 100644 --- a/src/search/engines/duckduckgo/suggest.go +++ b/src/search/engines/duckduckgo/suggest.go @@ -38,7 +38,7 @@ func (se Engine) Suggest(query string, locale options.Locale, sugChan chan resul Strs("suggestions", suggs). Msg("Sending suggestions to channel") for i, sug := range suggs { - sugChan <- result.NewSuggestionScraped(sug, se.Name, i) + sugChan <- result.NewSuggestionScraped(sug, se.Name, i+1) } if !foundResults.Load() { foundResults.Store(true) diff --git a/src/search/engines/google/suggest.go b/src/search/engines/google/suggest.go index b9b6b1ee..f1b8361c 100644 --- a/src/search/engines/google/suggest.go +++ b/src/search/engines/google/suggest.go @@ -38,7 +38,7 @@ func (se Engine) Suggest(query string, locale options.Locale, sugChan chan resul Strs("suggestions", suggs). Msg("Sending suggestions to channel") for i, sug := range suggs { - sugChan <- result.NewSuggestionScraped(sug, se.Name, i) + sugChan <- result.NewSuggestionScraped(sug, se.Name, i+1) } if !foundResults.Load() { foundResults.Store(true) diff --git a/src/search/result/rank/interfaces.go b/src/search/result/rank/interfaces.go new file mode 100644 index 00000000..d81b022a --- /dev/null +++ b/src/search/result/rank/interfaces.go @@ -0,0 +1,20 @@ +package rank + +import ( + "github.com/hearchco/agent/src/search/engines" +) + +type scoreRanker interface { + Score() float64 +} + +type scoreEngineRanker[T ranker] interface { + scoreRanker + + EngineRanks() []T +} + +type ranker interface { + SearchEngine() engines.Name + Rank() int +} diff --git a/src/search/result/rank/rank.go b/src/search/result/rank/results.go similarity index 50% rename from src/search/result/rank/rank.go rename to src/search/result/rank/results.go index f21a304b..7bcec95a 100644 --- a/src/search/result/rank/rank.go +++ b/src/search/result/rank/results.go @@ -9,25 +9,8 @@ import ( type Results []result.Result -// Cast []result.Result to Results, call rank() and return []result.Result slice. -func Rank(results []result.Result, rconf config.CategoryRanking) []result.Result { - resType := make(Results, 0, len(results)) - for _, res := range results { - resType = append(resType, res) - } - - // Rank the results. - resType.rank(rconf) - - rankedRes := make([]result.Result, 0, len(resType)) - for _, res := range resType { - rankedRes = append(rankedRes, res) - } - return rankedRes -} - // Calculates the Score, sorts by it and then populates the Rank field of every result. -func (r Results) rank(rconf config.CategoryRanking) { +func (r Results) Rank(rconf config.CategoryRanking) { // Fill Rank field for every EngineRank. r.fillEngineRankRank() @@ -35,7 +18,7 @@ func (r Results) rank(rconf config.CategoryRanking) { r.calculateScores(rconf) // Sort slice by score. - sort.Sort(ByScore(r)) + sort.Sort(ByScore[result.Result](r)) // Set correct ranks, by iterating over the sorted slice. r.correctRanks() diff --git a/src/search/result/rank/score.go b/src/search/result/rank/score.go index a30c15d2..20193901 100644 --- a/src/search/result/rank/score.go +++ b/src/search/result/rank/score.go @@ -4,7 +4,6 @@ import ( "math" "github.com/hearchco/agent/src/config" - "github.com/hearchco/agent/src/search/result" ) // Calculates and sets scores for all results. @@ -14,16 +13,24 @@ func (r Results) calculateScores(rconf config.CategoryRanking) { } } +// Calculates and sets scores for all results. +func (s Suggestions) calculateScores(rconf config.CategoryRanking) { + for i := range s { + sug := &s[i] + sug.SetScore(calculateScore(sug, rconf)) + } +} + // Calculates the score for one result. -func calculateScore(res result.Result, rconf config.CategoryRanking) float64 { +func calculateScore[T ranker](val scoreEngineRanker[T], rconf config.CategoryRanking) float64 { var retRankScore float64 = 0 - for _, er := range res.EngineRanks() { - eng := rconf.Engines[er.SearchEngine().ToLower()] + for _, er := range val.EngineRanks() { + eng := rconf.Engines[er.SearchEngine().String()] retRankScore += (100.0/math.Pow(float64(er.Rank())*rconf.A+rconf.B, rconf.REXP)*rconf.C+rconf.D)*eng.Mul + eng.Const } - retRankScore /= float64(len(res.EngineRanks())) - timesReturnedScore := math.Log(float64(len(res.EngineRanks()))*rconf.TRA+rconf.TRB)*10*rconf.TRC + rconf.TRD + retRankScore /= float64(len(val.EngineRanks())) + timesReturnedScore := math.Log(float64(len(val.EngineRanks()))*rconf.TRA+rconf.TRB)*10*rconf.TRC + rconf.TRD score := retRankScore + timesReturnedScore return score diff --git a/src/search/result/rank/sorting.go b/src/search/result/rank/sort.go similarity index 75% rename from src/search/result/rank/sorting.go rename to src/search/result/rank/sort.go index 9809803f..2dd29e0f 100644 --- a/src/search/result/rank/sorting.go +++ b/src/search/result/rank/sort.go @@ -8,11 +8,11 @@ import ( "github.com/hearchco/agent/src/search/result" ) -type ByScore []result.Result +type ByScore[T scoreRanker] []T -func (r ByScore) Len() int { return len(r) } -func (r ByScore) Swap(i, j int) { r[i], r[j] = r[j], r[i] } -func (r ByScore) Less(i, j int) bool { return r[i].Score() > r[j].Score() } +func (r ByScore[T]) Len() int { return len(r) } +func (r ByScore[T]) Swap(i, j int) { r[i], r[j] = r[j], r[i] } +func (r ByScore[T]) Less(i, j int) bool { return r[i].Score() > r[j].Score() } type ByPageAndOnPageRank []*result.Rank diff --git a/src/search/result/rank/suggestions.go b/src/search/result/rank/suggestions.go new file mode 100644 index 00000000..ebc212be --- /dev/null +++ b/src/search/result/rank/suggestions.go @@ -0,0 +1,29 @@ +package rank + +import ( + "sort" + + "github.com/hearchco/agent/src/config" + "github.com/hearchco/agent/src/search/result" +) + +type Suggestions []result.Suggestion + +// Calculates the Score, sorts by it and then populates the Rank field of every result. +func (s Suggestions) Rank(rconf config.CategoryRanking) { + // Calculate and set scores. + s.calculateScores(rconf) + + // Sort slice by score. + sort.Sort(ByScore[result.Suggestion](s)) + + // Set correct ranks, by iterating over the sorted slice. + s.correctRanks() +} + +func (s Suggestions) correctRanks() { + for i := range s { + sug := &s[i] + sug.SetRank(i + 1) + } +} diff --git a/src/search/result/suggestion.go b/src/search/result/suggestion.go index cd282a75..dea1c8a6 100644 --- a/src/search/result/suggestion.go +++ b/src/search/result/suggestion.go @@ -10,6 +10,8 @@ type Suggestion struct { type suggestionJSON struct { Value string `json:"value"` + Rank int `json:"rank"` + Score float64 `json:"score"` EngineRanks []RankSimple `json:"engine_ranks"` } @@ -17,6 +19,22 @@ func (s Suggestion) Value() string { return s.suggestionJSON.Value } +func (s Suggestion) Rank() int { + return s.suggestionJSON.Rank +} + +func (s *Suggestion) SetRank(rank int) { + s.suggestionJSON.Rank = rank +} + +func (s Suggestion) Score() float64 { + return s.suggestionJSON.Score +} + +func (s *Suggestion) SetScore(score float64) { + s.suggestionJSON.Score = score +} + func (s Suggestion) EngineRanks() []RankSimple { if s.suggestionJSON.EngineRanks == nil { log.Panic().Msg("EngineRanks is nil") diff --git a/src/search/result/suggestion_scraped.go b/src/search/result/suggestion_scraped.go index 9512dfea..62e8c018 100644 --- a/src/search/result/suggestion_scraped.go +++ b/src/search/result/suggestion_scraped.go @@ -34,8 +34,8 @@ func (s SuggestionScraped) Convert(erCap int) Suggestion { engineRanks = append(engineRanks, s.Rank().Convert()) return Suggestion{ suggestionJSON{ - s.Value(), - engineRanks, + Value: s.Value(), + EngineRanks: engineRanks, }, } }