diff --git a/cmd/newsletter/main.go b/cmd/newsletter/main.go index 1a1eb98..d7bdd5f 100644 --- a/cmd/newsletter/main.go +++ b/cmd/newsletter/main.go @@ -24,8 +24,8 @@ type Config struct { func main() { cfg := Config{ - LogLevel: getEnvWithDefault("LOG_LEVEL", "INFO"), - LogType: getEnvWithDefault("LOG_TYPE", "json"), + LogLevel: getEnvWithDefault("LOG_LEVEL", ""), + LogType: getEnvWithDefault("LOG_TYPE", ""), Mongo: mongodb.Config{ URI: getEnvWithDefault("NL_MONGO_URI", ""), }, diff --git a/docker-compose.yml b/docker-compose.yml index 961a3a9..3f8e09d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,8 @@ services: GOLANGCI_LINT_VERSION: $GOLANGCI_LINT_VERSION environment: NL_MONGO_URI: "mongodb://root:root@mongodb:27017" + LOG_LEVEL: "DEBUG" + LOG_TYPE: "json" depends_on: - mongodb volumes: diff --git a/scrape.go b/scrape.go index 0828f66..1b4b765 100644 --- a/scrape.go +++ b/scrape.go @@ -21,7 +21,7 @@ type PageContent struct { URL string } -// Storage is the interface that wraps the basic methods to save data and get data from the database +// Storage is the interface that wraps the basic methods to save and get data from the database type Storage interface { SaveSite(ctx context.Context, site []mongodb.Site) error DistinctEngineerURLs(ctx context.Context) ([]interface{}, error) @@ -50,7 +50,7 @@ func NewCrawler(maxJobs int, s time.Duration, signalCh chan os.Signal) *Crawler } } -// Run starts the crawler, s is the storage to save the results, f is the function to get the content of a url +// Run starts the crawler, where s represents the storage and f the function to fetch the content of a website func (c *Crawler) Run(ctx context.Context, s Storage, f func(string) (string, error)) { c.wg.Add(c.MaxJobs) for i := 0; i < c.MaxJobs; i++ { @@ -60,14 +60,14 @@ func (c *Crawler) Run(ctx context.Context, s Storage, f func(string) (string, er go func() { defer close(c.URLch) for range time.Tick(c.scheduler) { - slog.Info("fetching engineers") + slog.Debug("fetching engineers") gotURLs, err := s.DistinctEngineerURLs(ctx) if err != nil { slog.Error("error getting engineers", "error", err) c.signalCh <- syscall.SIGTERM } - slog.Info("fetched engineers", "engineers", len(gotURLs)) + slog.Debug("fetched engineers", "engineers", len(gotURLs)) for _, url := range gotURLs { c.URLch <- url.(string) } @@ -81,7 +81,7 @@ func (c *Crawler) Run(ctx context.Context, s Storage, f func(string) (string, er go func() { for v := range c.resultCh { - slog.Info("saving fetched sites response") + slog.Debug("saving fetched sites response") err := s.SaveSite(ctx, []mongodb.Site{ { URL: v.URL, diff --git a/scrape_test.go b/scrape_test.go index dccd6ce..fee07b7 100644 --- a/scrape_test.go +++ b/scrape_test.go @@ -13,6 +13,8 @@ import ( const fakeURL = "http://fakeurl.test" +// Even not verifying the result, this test is useful to check if the crawler is running properly, since it is +// using Mocks for the Storage and the Fetch function. func TestCrawlerRun(t *testing.T) { timeoutCh := time.After(time.Duration(150) * time.Millisecond) ctx := context.Background() @@ -73,6 +75,7 @@ func TestGetReferences_Status500(t *testing.T) { } } +// TODO: Move the StorageMock to a separate file, preferable in the same package(mongodb) type StorageMock interface { SaveSite(ctx context.Context, site []mongodb.Site) error DistinctEngineerURLs(ctx context.Context) ([]interface{}, error) @@ -85,10 +88,10 @@ func NewStorageMock() StorageMock { return StorageMockImpl{} } -func (s StorageMockImpl) SaveSite(ctx context.Context, site []mongodb.Site) error { +func (s StorageMockImpl) SaveSite(_ context.Context, _ []mongodb.Site) error { return nil } -func (s StorageMockImpl) DistinctEngineerURLs(ctx context.Context) ([]interface{}, error) { +func (s StorageMockImpl) DistinctEngineerURLs(_ context.Context) ([]interface{}, error) { return []interface{}{fakeURL}, nil }