Merge pull request #455 from hearchco/as/fix/construct-url

fix(result): construct result check for proper url
hearchco · Nov 21, 2024 · f0829d3 · f0829d3
2 parents c7575cc + d0c3031
commit f0829d3
Show file tree

Hide file tree

Showing 16 changed files with 86 additions and 2 deletions.
diff --git a/src/search/engines/bing/s_images.go b/src/search/engines/bing/s_images.go
@@ -208,6 +208,11 @@ func (se Engine) ImageSearch(query string, opts options.Options, resChan chan re
 			log.Error().
 				Caller().
 				Err(err).
+				Str("url", jsonMetadata.ImageURL).
+				Str("title", titleText).
+				Str("desc", jsonMetadata.Desc).
+				Int("page", page).
+				Int("rank", pageRankCounter.GetPlusOne(pageIndex)).
 				Msg("Failed to construct result")
 		} else {
 			log.Trace().

diff --git a/src/search/engines/bing/s_web.go b/src/search/engines/bing/s_web.go
@@ -55,6 +55,11 @@ func (se Engine) WebSearch(query string, opts options.Options, resChan chan resu
 			log.Error().
 				Caller().
 				Err(err).
+				Str("url", urlText).
+				Str("title", titleText).
+				Str("desc", descText).
+				Int("page", page).
+				Int("rank", pageRankCounter.GetPlusOne(pageIndex)).
 				Msg("Failed to construct result")
 		} else {
 			log.Trace().

diff --git a/src/search/engines/brave/s_web.go b/src/search/engines/brave/s_web.go
@@ -46,6 +46,11 @@ func (se Engine) WebSearch(query string, opts options.Options, resChan chan resu
 			log.Error().
 				Caller().
 				Err(err).
+				Str("url", urlText).
+				Str("title", titleText).
+				Str("desc", descText).
+				Int("page", page).
+				Int("rank", pageRankCounter.GetPlusOne(pageIndex)).
 				Msg("Failed to construct result")
 		} else {
 			log.Trace().

diff --git a/src/search/engines/duckduckgo/s_web.go b/src/search/engines/duckduckgo/s_web.go
@@ -72,6 +72,11 @@ func (se Engine) WebSearch(query string, opts options.Options, resChan chan resu
 					log.Error().
 						Caller().
 						Err(err).
+						Str("url", urlText).
+						Str("title", titleText).
+						Str("desc", descText).
+						Int("page", page).
+						Int("rank", onPageRank).
 						Msg("Failed to construct result")
 				} else {
 					log.Trace().

diff --git a/src/search/engines/etools/s_web.go b/src/search/engines/etools/s_web.go
@@ -56,6 +56,11 @@ func (se Engine) WebSearch(query string, opts options.Options, resChan chan resu
 			log.Error().
 				Caller().
 				Err(err).
+				Str("url", urlText).
+				Str("title", titleText).
+				Str("desc", descText).
+				Int("page", page).
+				Int("rank", pageRankCounter.GetPlusOne(pageIndex)).
 				Msg("Failed to construct result")
 		} else {
 			log.Trace().

diff --git a/src/search/engines/google/s_images.go b/src/search/engines/google/s_images.go
@@ -70,7 +70,11 @@ func (se Engine) ImageSearch(query string, opts options.Options, resChan chan re
 					log.Error().
 						Caller().
 						Err(err).
-						Str("result", fmt.Sprintf("%v", r)).
+						Str("url", origImg.Url).
+						Str("title", resultJson.PageTitle).
+						Str("desc", textInGridJson.Snippet).
+						Int("page", page).
+						Int("rank", pageRankCounter.GetPlusOne(pageIndex)).
 						Msg("Failed to construct result")
 				} else {
 					log.Trace().

diff --git a/src/search/engines/google/s_web.go b/src/search/engines/google/s_web.go
@@ -32,6 +32,11 @@ func (se Engine) WebSearch(query string, opts options.Options, resChan chan resu
 			log.Error().
 				Caller().
 				Err(err).
+				Str("url", urlText).
+				Str("title", titleText).
+				Str("desc", descText).
+				Int("page", page).
+				Int("rank", pageRankCounter.GetPlusOne(pageIndex)).
 				Msg("Failed to construct result")
 		} else {
 			log.Trace().

diff --git a/src/search/engines/googlescholar/s_web.go b/src/search/engines/googlescholar/s_web.go
@@ -45,6 +45,11 @@ func (se Engine) WebSearch(query string, opts options.Options, resChan chan resu
 			log.Error().
 				Caller().
 				Err(err).
+				Str("url", urlText).
+				Str("title", titleText).
+				Str("desc", descText).
+				Int("page", page).
+				Int("rank", pageRankCounter.GetPlusOne(pageIndex)).
 				Msg("Failed to construct result")
 		} else {
 			log.Trace().

diff --git a/src/search/engines/mojeek/s_web.go b/src/search/engines/mojeek/s_web.go
@@ -32,6 +32,11 @@ func (se Engine) WebSearch(query string, opts options.Options, resChan chan resu
 			log.Error().
 				Caller().
 				Err(err).
+				Str("url", urlText).
+				Str("title", titleText).
+				Str("desc", descText).
+				Int("page", page).
+				Int("rank", pageRankCounter.GetPlusOne(pageIndex)).
 				Msg("Failed to construct result")
 		} else {
 			log.Trace().

diff --git a/src/search/engines/presearch/s_web.go b/src/search/engines/presearch/s_web.go
@@ -64,6 +64,11 @@ func (se Engine) WebSearch(query string, opts options.Options, resChan chan resu
 					log.Error().
 						Caller().
 						Err(err).
+						Str("url", goodURL).
+						Str("title", goodTitle).
+						Str("desc", goodDesc).
+						Int("page", page).
+						Int("rank", counter).
 						Msg("Failed to construct result")
 				} else {
 					log.Trace().

diff --git a/src/search/engines/qwant/s_web.go b/src/search/engines/qwant/s_web.go
@@ -54,6 +54,11 @@ func (se Engine) WebSearch(query string, opts options.Options, resChan chan resu
 					log.Error().
 						Caller().
 						Err(err).
+						Str("url", goodURL).
+						Str("title", goodTitle).
+						Str("desc", goodDesc).
+						Int("page", page).
+						Int("rank", counter).
 						Msg("Failed to construct result")
 				} else {
 					log.Trace().

diff --git a/src/search/engines/startpage/s_web.go b/src/search/engines/startpage/s_web.go
@@ -33,6 +33,11 @@ func (se Engine) WebSearch(query string, opts options.Options, resChan chan resu
 			log.Error().
 				Caller().
 				Err(err).
+				Str("url", urlText).
+				Str("title", titleText).
+				Str("desc", descText).
+				Int("page", page).
+				Int("rank", pageRankCounter.GetPlusOne(pageIndex)).
 				Msg("Failed to construct result")
 		} else {
 			log.Trace().

diff --git a/src/search/engines/swisscows/s_web.go b/src/search/engines/swisscows/s_web.go
@@ -73,6 +73,11 @@ func (se Engine) WebSearch(query string, opts options.Options, resChan chan resu
 				log.Error().
 					Caller().
 					Err(err).
+					Str("url", goodURL).
+					Str("title", goodTitle).
+					Str("desc", goodDesc).
+					Int("page", page).
+					Int("rank", counter).
 					Msg("Failed to construct result")
 			} else {
 				log.Trace().

diff --git a/src/search/engines/yahoo/s_web.go b/src/search/engines/yahoo/s_web.go
@@ -74,6 +74,11 @@ func (se Engine) WebSearch(query string, opts options.Options, resChan chan resu
 			log.Error().
 				Caller().
 				Err(err).
+				Str("url", urlText).
+				Str("title", titleText).
+				Str("desc", descText).
+				Int("page", page).
+				Int("rank", pageRankCounter.GetPlusOne(pageIndex)).
 				Msg("Failed to construct result")
 		} else {
 			log.Trace().

diff --git a/src/search/engines/yep/s_web.go b/src/search/engines/yep/s_web.go
@@ -74,6 +74,11 @@ package yep
 // 				log.Error().
 // 					Caller().
 // 					Err(err).
+// 					Str("url", goodURL).
+// 					Str("title", goodTitle).
+// 					Str("desc", goodDesc).
+// 					Int("page", page).
+// 					Int("rank", pageRankCounter.GetPlusOne(pageIndex)).
 // 					Msg("Failed to construct result")
 // 			} else {
 // 				log.Trace().

diff --git a/src/search/result/construct.go b/src/search/result/construct.go
@@ -2,6 +2,7 @@ package result
 
 import (
 	"fmt"
+	"net/url"
 
 	"github.com/hearchco/agent/src/search/engines"
 )
@@ -11,6 +12,15 @@ func ConstructResult(seName engines.Name, urll string, title string, description
 		return WebScraped{}, fmt.Errorf("invalid URL: empty")
 	}
 
+	u, err := url.Parse(urll)
+	if err != nil {
+		return WebScraped{}, fmt.Errorf("invalid URL: %s", err)
+	}
+
+	if u.Hostname() == "" {
+		return WebScraped{}, fmt.Errorf("invalid URL: no hostname")
+	}
+
 	if title == "" {
 		return WebScraped{}, fmt.Errorf("invalid title: empty")
 	}
@@ -24,7 +34,7 @@ func ConstructResult(seName engines.Name, urll string, title string, description
 	}
 
 	return WebScraped{
-		url:         urll,
+		url:         u.String(),
 		title:       title,
 		description: description,
 		rank: RankScraped{