Merge pull request #333 from hearchco/as/fix/tiny

fix: switch to consts instead of runtime structs and comments

src/config/defaults_cat_general.go

@@ -29,7 +29,7 @@ var generalRequiredByOriginEngines = []engines.Name{
 }
 
 var generalPreferredEngines = []engines.Name{
-	engines.ETOOLS,
+	engines.ETOOLS, // Not in ByOrigin because it only gives 10 results across a lot of engines that it scrapes from.
 }
 
 var generalPreferredByOriginEngines = []engines.Name{

src/search/engines/bing/info.go

@@ -0,0 +1,12 @@
+package bing
+
+import (
+	"github.com/hearchco/agent/src/search/engines"
+)
+
+const (
+	seName    = engines.BING
+	searchURL = "https://www.bing.com/search"
+)
+
+var origins = [...]engines.Name{seName}

src/search/engines/bing/params.go

@@ -7,7 +7,14 @@ import (
 	"github.com/hearchco/agent/src/search/engines/options"
 )
 
+const (
+	paramKeyPage       = "first"
+	paramKeyLocale     = "setlang" // Should be first 2 characters of Locale.
+	paramKeyLocaleSec  = "cc"      // Should be last 2 characters of Locale.
+	paramKeySafeSearch = ""        // Always enabled.
+)
+
 func localeParamString(locale options.Locale) string {
 	spl := strings.SplitN(strings.ToLower(locale.String()), "_", 2)
-	return fmt.Sprintf("%v=%v&%v=%v", params.Locale, spl[0], params.LocaleSec, spl[1])
+	return fmt.Sprintf("%v=%v&%v=%v", paramKeyLocale, spl[0], paramKeyLocaleSec, spl[1])
 }

src/search/engines/bing/search.go

@@ -21,10 +21,10 @@ type Engine struct {
 	scraper.EngineBase
 }
 
-func New() *Engine {
-	return &Engine{EngineBase: scraper.EngineBase{
-		Name:    info.Name,
-		Origins: info.Origins,
+func New() scraper.Enginer {
+	return &Engine{scraper.EngineBase{
+		Name:    seName,
+		Origins: origins[:],
 	}}
 }
 
@@ -84,23 +84,23 @@ func (se Engine) Search(query string, opts options.Options, resChan chan result.
 	})
 
 	// Static params.
-	localeParam := localeParamString(opts.Locale)
+	paramLocale := localeParamString(opts.Locale)
 
 	for i := range opts.Pages.Max {
 		pageNum0 := i + opts.Pages.Start
 		ctx := colly.NewContext()
 		ctx.Put("page", strconv.Itoa(i))
 
 		// Dynamic params.
-		pageParam := ""
+		paramPage := ""
 		if pageNum0 > 0 {
-			pageParam = fmt.Sprintf("%v=%v", params.Page, pageNum0*10+1)
+			paramPage = fmt.Sprintf("%v=%v", paramKeyPage, pageNum0*10+1)
 		}
 
-		combinedParams := morestrings.JoinNonEmpty([]string{pageParam, localeParam}, "&", "&")
+		combinedParams := morestrings.JoinNonEmpty([]string{paramPage, paramLocale}, "&", "&")
 
-		urll := fmt.Sprintf("%v?q=%v%v", info.URL, query, combinedParams)
-		anonUrll := fmt.Sprintf("%v?q=%v%v", info.URL, anonymize.String(query), combinedParams)
+		urll := fmt.Sprintf("%v?q=%v%v", searchURL, query, combinedParams)
+		anonUrll := fmt.Sprintf("%v?q=%v%v", searchURL, anonymize.String(query), combinedParams)
 
 		if err := se.Get(ctx, urll, anonUrll); err != nil {
 			retErrors = append(retErrors, err)

src/search/engines/bing/search_test.go

@@ -9,14 +9,11 @@ import (
 )
 
 func TestSearch(t *testing.T) {
-	// Search engine name
-	seName := info.Name
-
-	// testing options
+	// Testing options.
 	conf := _engines_test.NewConfig(seName)
 	opt := _engines_test.NewOpts()
 
-	// test cases
+	// Test cases.
 	tchar := []_engines_test.TestCaseHasAnyResults{{
 		Query:   "ping",
 		Options: opt,

src/search/engines/bingimages/info.go

@@ -0,0 +1,12 @@
+package bingimages
+
+import (
+	"github.com/hearchco/agent/src/search/engines"
+)
+
+const (
+	seName    = engines.BINGIMAGES
+	searchURL = "https://www.bing.com/images/async"
+)
+
+var origins = [...]engines.Name{seName}

src/search/engines/bingimages/params.go

@@ -7,7 +7,17 @@ import (
 	"github.com/hearchco/agent/src/search/engines/options"
 )
 
+const (
+	paramKeyPage       = "first"
+	paramKeyLocale     = "setlang" // Should be first 2 characters of Locale.
+	paramKeyLocaleSec  = "cc"      // Should be last 2 characters of Locale.
+	paramKeySafeSearch = ""        // Always enabled.
+
+	paramAsync = "async=1"
+	paramCount = "count=35"
+)
+
 func localeParamString(locale options.Locale) string {
 	spl := strings.SplitN(strings.ToLower(locale.String()), "_", 2)
-	return fmt.Sprintf("%v=%v&%v=%v", params.Locale, spl[0], params.LocaleSec, spl[1])
+	return fmt.Sprintf("%v=%v&%v=%v", paramKeyLocale, spl[0], paramKeyLocaleSec, spl[1])
 }

src/search/engines/bingimages/search.go

@@ -21,10 +21,10 @@ type Engine struct {
 	scraper.EngineBase
 }
 
-func New() *Engine {
-	return &Engine{EngineBase: scraper.EngineBase{
-		Name:    info.Name,
-		Origins: info.Origins,
+func New() scraper.Enginer {
+	return &Engine{scraper.EngineBase{
+		Name:    seName,
+		Origins: origins[:],
 	}}
 }
 
@@ -222,20 +222,20 @@ func (se Engine) Search(query string, opts options.Options, resChan chan result.
 	})
 
 	// Static params.
-	localeParam := localeParamString(opts.Locale)
+	paramLocale := localeParamString(opts.Locale)
 
 	for i := range opts.Pages.Max {
 		pageNum0 := i + opts.Pages.Start
 		ctx := colly.NewContext()
 		ctx.Put("page", strconv.Itoa(i))
 
 		// Dynamic params.
-		pageParam := fmt.Sprintf("%v=%v", params.Page, pageNum0*35+1)
+		paramPage := fmt.Sprintf("%v=%v", paramKeyPage, pageNum0*35+1)
 
-		combinedParams := morestrings.JoinNonEmpty([]string{asyncParam, pageParam, countParam, localeParam}, "&", "&")
+		combinedParams := morestrings.JoinNonEmpty([]string{paramAsync, paramPage, paramCount, paramLocale}, "&", "&")
 
-		urll := fmt.Sprintf("%v?q=%v%v", info.URL, query, combinedParams)
-		anonUrll := fmt.Sprintf("%v?q=%v%v", info.URL, anonymize.String(query), combinedParams)
+		urll := fmt.Sprintf("%v?q=%v%v", searchURL, query, combinedParams)
+		anonUrll := fmt.Sprintf("%v?q=%v%v", searchURL, anonymize.String(query), combinedParams)
 
 		if err := se.Get(ctx, urll, anonUrll); err != nil {
 			retErrors = append(retErrors, err)

src/search/engines/bingimages/search_test.go

@@ -9,14 +9,11 @@ import (
 )
 
 func TestSearch(t *testing.T) {
-	// Search engine name
-	seName := info.Name
-
-	// testing options
+	// Testing options.
 	conf := _engines_test.NewConfig(seName)
 	opt := _engines_test.NewOpts()
 
-	// test cases
+	// Test cases.
 	tchar := []_engines_test.TestCaseHasAnyResults{{
 		Query:   "ping",
 		Options: opt,

src/search/engines/brave/info.go

@@ -0,0 +1,12 @@
+package brave
+
+import (
+	"github.com/hearchco/agent/src/search/engines"
+)
+
+const (
+	seName    = engines.BRAVE
+	searchURL = "https://search.brave.com/search"
+)
+
+var origins = [...]engines.Name{seName, engines.GOOGLE}

src/search/engines/brave/params.go

@@ -0,0 +1,30 @@
+package brave
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/hearchco/agent/src/search/engines/options"
+)
+
+const (
+	paramKeyPage       = "offset"
+	paramKeyLocale     = "country"    // Should be last 2 characters of Locale.
+	paramKeySafeSearch = "safesearch" // Can be "off" or "strict".
+
+	paramSource     = "source=web"
+	paramSpellcheck = "spellcheck=0"
+)
+
+func localeCookieString(locale options.Locale) string {
+	region := strings.SplitN(strings.ToLower(locale.String()), "_", 2)[1]
+	return fmt.Sprintf("%v=%v", paramKeyLocale, region)
+}
+
+func safeSearchCookieString(safesearch bool) string {
+	if safesearch {
+		return fmt.Sprintf("%v=%v", paramKeySafeSearch, "strict")
+	} else {
+		return fmt.Sprintf("%v=%v", paramKeySafeSearch, "off")
+	}
+}

src/search/engines/brave/search.go

@@ -20,10 +20,10 @@ type Engine struct {
 	scraper.EngineBase
 }
 
-func New() *Engine {
-	return &Engine{EngineBase: scraper.EngineBase{
-		Name:    info.Name,
-		Origins: info.Origins,
+func New() scraper.Enginer {
+	return &Engine{scraper.EngineBase{
+		Name:    seName,
+		Origins: origins[:],
 	}}
 }
 
@@ -80,15 +80,15 @@ func (se Engine) Search(query string, opts options.Options, resChan chan result.
 		ctx.Put("page", strconv.Itoa(i))
 
 		// Dynamic params.
-		pageParam := ""
-		combinedParams := morestrings.JoinNonEmpty([]string{sourceParam, pageParam}, "&", "&")
+		paramPage := ""
+		combinedParams := morestrings.JoinNonEmpty([]string{paramSource, paramPage}, "&", "&")
 		if pageNum0 > 0 {
-			pageParam = fmt.Sprintf("%v=%v", params.Page, pageNum0)
-			combinedParams = morestrings.JoinNonEmpty([]string{spellcheckParam, pageParam}, "&", "&")
+			paramPage = fmt.Sprintf("%v=%v", paramKeyPage, pageNum0)
+			combinedParams = morestrings.JoinNonEmpty([]string{paramSpellcheck, paramPage}, "&", "&")
 		}
 
-		urll := fmt.Sprintf("%v?q=%v%v", info.URL, query, combinedParams)
-		anonUrll := fmt.Sprintf("%v?q=%v%v", info.URL, anonymize.String(query), combinedParams)
+		urll := fmt.Sprintf("%v?q=%v%v", searchURL, query, combinedParams)
+		anonUrll := fmt.Sprintf("%v?q=%v%v", searchURL, anonymize.String(query), combinedParams)
 
 		if err := se.Get(ctx, urll, anonUrll); err != nil {
 			retErrors = append(retErrors, err)

src/search/engines/brave/search_test.go

@@ -9,14 +9,11 @@ import (
 )
 
 func TestSearch(t *testing.T) {
-	// Search engine name
-	seName := info.Name
-
-	// testing options
+	// Testing options.
 	conf := _engines_test.NewConfig(seName)
 	opt := _engines_test.NewOpts()
 
-	// test cases
+	// Test cases.
 	tchar := []_engines_test.TestCaseHasAnyResults{{
 		Query:   "ping",
 		Options: opt,