feat: huge refactor

Makefile

@@ -1,17 +1,11 @@
 run:
 	air -- --pretty
-run-cli:
-	go run ./src --cli --pretty
 
 debug:
 	air -- --pretty -v
-debug-cli:
-	go run ./srv --cli --pretty -v
 
 trace:
 	air -- --pretty -vv
-trace-cli:
-	go run ./src --cli --pretty -vv
 
 install:
 	go get ./...

docker/Dockerfile

@@ -11,4 +11,4 @@ VOLUME [ "/data" ]
 
 EXPOSE 3030
 
-LABEL org.opencontainers.image.source="https://github.com/hearchco/hearchco"
+LABEL org.opencontainers.image.source="https://github.com/hearchco/agent"

generate/searcher/searcher.go

@@ -17,13 +17,15 @@ import (
 )
 
 var (
-	typeName      = flag.String("type", "", "type name; must be set")
-	output        = flag.String("output", "", "output file name; default srcdir/<type>_searcher.go")
-	trimprefix    = flag.String("trimprefix", "", "trim the `prefix` from the generated constant names")
-	buildTags     = flag.String("tags", "", "comma-separated list of build tags to apply")
-	packageName   = flag.String("packagename", "", "name of the package for generated code; default current package")
-	enginesImport = flag.String("enginesimport", "github.com/hearchco/hearchco/src/search/engines", "source of the engines import, which is prefixed to imports for consts; default github.com/hearchco/hearchco/src/search/engines")
-	linecomment   = flag.Bool("linecomment", false, "use line comment text as printed text when present")
+	typeName        = flag.String("type", "", "type name; must be set")
+	output          = flag.String("output", "", "output file name; default srcdir/<type>_searcher.go")
+	trimprefix      = flag.String("trimprefix", "", "trim the `prefix` from the generated constant names")
+	buildTags       = flag.String("tags", "", "comma-separated list of build tags to apply")
+	packageName     = flag.String("packagename", "", "name of the package for generated code; default current package")
+	interfaceImport = flag.String("interfaceimport", "github.com/hearchco/agent/src/search/scraper", "source of the interface import, which is prefixed to interfaces; default github.com/hearchco/agent/src/search/scraper")
+	interfaceName   = flag.String("interfacename", "scraper.Enginer", "name of the interface; default scraper.Enginer")
+	enginesImport   = flag.String("enginesimport", "github.com/hearchco/agent/src/search/engines", "source of the engines import, which is prefixed to imports for engines; default github.com/hearchco/agent/src/search/engines")
+	linecomment     = flag.Bool("linecomment", false, "use line comment text as printed text when present")
 )
 
 // Usage is a replacement usage function for the flags package.
@@ -90,7 +92,8 @@ func main() {
 	}
 	g.Printf("package %s", pkgName)
 	g.Printf("\n")
-	g.Printf("import \"%s\"\n", *enginesImport) // Used by all methods.
+	g.Printf("import \"%s\"\n", *interfaceImport)
+	g.Printf("import \"%s\"\n", *enginesImport)
 
 	// Run generate for each type.
 	for _, typeName := range types {
@@ -195,7 +198,7 @@ func (g *Generator) generate(typeName string) {
 	}
 	g.Printf("}\n")
 
-	g.buildOneRun(values)
+	g.printEnginer(values, *interfaceName)
 }
 
 // format returns the gofmt-ed contents of the Generator's buffer.
@@ -314,17 +317,19 @@ func (f *File) genDecl(node ast.Node) bool {
 	return false
 }
 
-// buildOneRun generates the variables and NewEngineStarter func for a single run of contiguous values.
-func (g *Generator) buildOneRun(values []Value) {
+func (g *Generator) printEnginer(values []Value, interfaceName string) {
 	g.Printf("\n")
-	// The generated code is simple enough to write as a Printf format.
-	g.Printf("\nfunc NewEngineStarter() [%d]Searcher {", len(values))
-	g.Printf("\n\tvar engineArray [%d]Searcher", len(values))
+	g.Printf("\nfunc enginerArray() [%d]%s {", len(values), interfaceName)
+	g.Printf("\n\tvar engineArray [%d]%s", len(values), interfaceName)
 	for _, v := range values {
 		if validConst(v) {
 			g.Printf("\n\tengineArray[%s.%s] = %s.New()", g.pkg.name, v.name, strings.ToLower(v.name))
 		}
 	}
 	g.Printf("\n\treturn engineArray")
 	g.Printf("\n}")
+
+	g.Printf("\n")
+	g.Printf("\nconst enginerLen = %d", len(values))
+	g.Printf("\n")
 }

go.mod

@@ -1,78 +1,61 @@
-module github.com/hearchco/hearchco
+module github.com/hearchco/agent
 
 go 1.22
 
 toolchain go1.22.0
 
 require (
+	github.com/PuerkitoBio/goquery v1.9.2
 	github.com/alecthomas/kong v0.9.0
 	github.com/andybalholm/brotli v1.1.0
 	github.com/aws/aws-lambda-go v1.47.0
 	github.com/awslabs/aws-lambda-go-api-proxy v0.16.2
-	github.com/dgraph-io/badger/v4 v4.2.0
 	github.com/go-chi/chi/v5 v5.0.12
 	github.com/go-chi/cors v1.2.1
-	github.com/gocolly/colly/v2 v2.1.1-0.20231020184023-3c987f1982ed
+	github.com/gocolly/colly/v2 v2.1.1-0.20231020184023-3c987f1982ed // Hearchco's PR
 	github.com/knadh/koanf/parsers/yaml v0.1.0
 	github.com/knadh/koanf/providers/env v0.1.0
 	github.com/knadh/koanf/providers/file v0.1.0
 	github.com/knadh/koanf/providers/structs v0.1.0
 	github.com/knadh/koanf/v2 v2.1.1
 	github.com/pkg/profile v1.7.0
-	github.com/redis/go-redis/v9 v9.5.2
+	github.com/redis/go-redis/v9 v9.5.3
 	github.com/rs/zerolog v1.33.0
+	golang.org/x/net v0.26.0
 	golang.org/x/tools v0.22.0
 )
 
 require (
+	github.com/andybalholm/cascadia v1.3.2 // indirect
+	github.com/antchfx/htmlquery v1.3.1 // indirect
+	github.com/antchfx/xmlquery v1.4.0 // indirect
+	github.com/antchfx/xpath v1.3.0 // indirect
 	github.com/bits-and-blooms/bitset v1.13.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/dgraph-io/ristretto v0.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/fatih/structs v1.1.0 // indirect
 	github.com/felixge/fgprof v0.9.4 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
-	github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/glog v1.2.1 // indirect
-	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/flatbuffers v24.3.25+incompatible // indirect
-	github.com/google/pprof v0.0.0-20240528025155-186aa0362fba // indirect
-	github.com/klauspost/compress v1.17.8 // indirect
-	github.com/knadh/koanf/maps v0.1.1 // indirect
-	github.com/kr/pretty v0.3.1 // indirect
-	github.com/mitchellh/copystructure v1.2.0 // indirect
-	github.com/mitchellh/reflectwalk v1.0.2 // indirect
-	github.com/nlnwa/whatwg-url v0.4.1 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
-	github.com/rogpeppe/go-internal v1.11.0 // indirect
-	github.com/rs/xid v1.5.0 // indirect
-	github.com/stretchr/testify v1.8.4 // indirect
-	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/mod v0.18.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
-)
-
-require (
-	github.com/PuerkitoBio/goquery v1.9.2
-	github.com/andybalholm/cascadia v1.3.2 // indirect
-	github.com/antchfx/htmlquery v1.3.1 // indirect
-	github.com/antchfx/xmlquery v1.4.0 // indirect
-	github.com/antchfx/xpath v1.3.0 // indirect
+	github.com/go-viper/mapstructure/v2 v2.0.0 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/pprof v0.0.0-20240528025155-186aa0362fba // indirect
 	github.com/kennygrant/sanitize v1.2.4 // indirect
+	github.com/knadh/koanf/maps v0.1.1 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/nlnwa/whatwg-url v0.4.1 // indirect
+	github.com/rs/xid v1.5.0 // indirect
 	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
 	github.com/temoto/robotstxt v1.1.2 // indirect
-	golang.org/x/net v0.26.0
+	golang.org/x/mod v0.18.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
 	golang.org/x/sys v0.21.0 // indirect
 	golang.org/x/text v0.16.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
-	google.golang.org/protobuf v1.34.1 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )