From 37699eaf22393579a2d8bf19ba37d118b78fe6cd Mon Sep 17 00:00:00 2001 From: jojo Date: Thu, 4 Jan 2024 21:39:55 -0300 Subject: [PATCH] :sparkles: nl basic operations --- Makefile | 2 +- mongodb/db.go | 15 ++++- mongodb/db_test.go | 51 ---------------- mongodb/newsletter.go | 65 ++++++++++++++++++++ mongodb/newsletter_test.go | 118 +++++++++++++++++++++++++++++++++++++ scrape_test.go | 11 ---- 6 files changed, 196 insertions(+), 66 deletions(-) delete mode 100644 mongodb/db_test.go create mode 100644 mongodb/newsletter.go create mode 100644 mongodb/newsletter_test.go delete mode 100644 scrape_test.go diff --git a/Makefile b/Makefile index 0164eab..c55cc84 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ test: .PHONY: integration-test integration-test: - go test -timeout 5s -tags=integration ./... + go test -timeout 5s -tags=integration ./... -v ## builds the service .PHONY: service diff --git a/mongodb/db.go b/mongodb/db.go index b4583e9..27bc986 100644 --- a/mongodb/db.go +++ b/mongodb/db.go @@ -13,12 +13,21 @@ type Config struct { URI string } -// Connect connects to the MongoDB instance. -func Connect(ctx context.Context, cfg Config) (*mongo.Client, error) { - client, err := mongo.Connect(ctx, options.Client().ApplyURI(cfg.URI)) +// OpenDB connects to the MongoDB instance. +func OpenDB(ctx context.Context, cfg Config) (*mongo.Client, error) { + bsonOpts := &options.BSONOptions{ + UseJSONStructTags: true, + NilSliceAsEmpty: true, + } + + client, err := mongo.Connect(ctx, options.Client(). + ApplyURI(cfg.URI). + SetBSONOptions(bsonOpts)) + if err != nil { return nil, err } + err = client.Ping(ctx, nil) if err != nil { return nil, err diff --git a/mongodb/db_test.go b/mongodb/db_test.go deleted file mode 100644 index 3aa30e6..0000000 --- a/mongodb/db_test.go +++ /dev/null @@ -1,51 +0,0 @@ -//go:build integration -// +build integration - -package mongodb - -import ( - "context" - "testing" - - "go.mongodb.org/mongo-driver/mongo" -) - -func setup(ctx context.Context, t testing.TB) (*mongo.Collection, func()) { - client, err := Connect(ctx, Config{ - URI: "mongodb://root:root@mongodb:27017", - }) - - if err != nil { - panic(err) - } - - randCollection := t.Name() - - collection := client.Database("test").Collection(randCollection) - - teardown := func() { - err = collection.Drop(ctx) - if err != nil { - panic(err) - } - } - - return collection, teardown -} - -func TestConnect(t *testing.T) { - collection, teardown := setup(context.Background(), t) - - defer teardown() - - resp, err := collection.InsertOne(context.Background(), map[string]string{"name": "pi", "value": "3.14159"}) - - if err != nil { - t.Fatal(err) - } - t.Log(resp.InsertedID) - - result := collection.FindOne(context.Background(), map[string]string{"name": "pi", "value": "3.14159"}) - t.Log(result.Raw()) - -} diff --git a/mongodb/newsletter.go b/mongodb/newsletter.go new file mode 100644 index 0000000..acea5f7 --- /dev/null +++ b/mongodb/newsletter.go @@ -0,0 +1,65 @@ +package mongodb + +import ( + "context" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" +) + +// Newsletter is the struct that gather what websites to scrape for an user email +type Newsletter struct { + UserEmail string `bson:"user_email"` + URLs []string `bson:"urls"` +} + +// Site is the struct that gather the scraped content of a website +type Site struct { + UserEmail string `bson:"user_email"` + URL string `bson:"url"` + Content string `bson:"content"` + ScrapeDate time.Time `bson:"scrape_date"` +} + +// NLStorage joins the Mongo operations for the Newsletter collection +type NLStorage struct { + client *mongo.Client + DBName string +} + +// NewNLStorage initializes a new NLStorage +func NewNLStorage(client *mongo.Client, DBName string) *NLStorage { + return &NLStorage{ + client: client, + DBName: DBName, + } +} + +// SaveNewsletter saves a newsletter in the database +func (m *NLStorage) SaveNewsletter(ctx context.Context, newsletter Newsletter) error { + database := m.client.Database(m.DBName) + collection := database.Collection("newsletter") + _, err := collection.InsertOne(ctx, newsletter) + if err != nil { + return err + } + return nil +} + +// Newsletter returns all the newsletters in the database +func (m *NLStorage) Newsletter() ([]Newsletter, error) { + var newsletters []Newsletter + database := m.client.Database(m.DBName) + collection := database.Collection("newsletter") + cursor, err := collection.Find(context.Background(), bson.M{}) + if err != nil { + return nil, err + } + + if err = cursor.All(context.Background(), &newsletters); err != nil { + return nil, err + } + + return newsletters, nil +} diff --git a/mongodb/newsletter_test.go b/mongodb/newsletter_test.go new file mode 100644 index 0000000..d158c36 --- /dev/null +++ b/mongodb/newsletter_test.go @@ -0,0 +1,118 @@ +//go:build integration +// +build integration + +package mongodb + +import ( + "context" + "fmt" + "reflect" + "testing" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" +) + +func TestNLStorageSaveNewsletter(t *testing.T) { + ctx := context.Background() + client, DBName := setup(ctx, t) + + database := client.Database(DBName) + collection := database.Collection("newsletter") + + NLStorage := NewNLStorage(client, DBName) + err := NLStorage.SaveNewsletter(ctx, Newsletter{ + UserEmail: "j@gmail.com", + URLs: []string{"https://www.google.com"}, + }) + + if err != nil { + t.Fatal("error saving newsletter", err) + } + + var nls []Newsletter + cursor, err := collection.Find(context.Background(), bson.M{}) + + if err != nil { + t.Fatal("error finding newsletter", err) + } + + if err := cursor.All(ctx, &nls); err != nil { + t.Fatal("error decoding newsletter", err) + } + + if len(nls) == 1 { + reflect.DeepEqual(nls[0], Newsletter{ + UserEmail: "j@gmail.com", + URLs: []string{"https://www.google.com"}, + }) + } else { + t.Fatal("expected 1 newsletter, got", len(nls)) + } + + t.Cleanup(teardown(ctx, client, DBName)) +} + +func TestNLStorageNewsletter(t *testing.T) { + ctx := context.Background() + client, DBName := setup(ctx, t) + + database := client.Database(DBName) + collection := database.Collection("newsletter") + + _, err := collection.InsertOne(ctx, Newsletter{ + UserEmail: "j@gmail.com", + URLs: []string{"https://www.google.com"}, + }) + + if err != nil { + t.Fatal("error saving newsletter", err) + } + + NLStorage := NewNLStorage(client, DBName) + nls, err := NLStorage.Newsletter() + if err != nil { + t.Fatal("error getting newsletter", err) + } + + if len(nls) == 1 { + reflect.DeepEqual(nls[0], Newsletter{ + UserEmail: "j@gmail.com", + URLs: []string{"https://www.google.com"}, + }) + } else { + t.Fatal("expected 1 newsletter, got", len(nls)) + } + + t.Cleanup(teardown(ctx, client, DBName)) +} + +func assert(t testing.TB, got, want interface{}) { + t.Helper() + if got != want { + t.Fatalf("got %v, want %v", got, want) + } +} + +func teardown(ctx context.Context, client *mongo.Client, DBName string) func() { + return func() { + if err := client.Database(DBName).Drop(ctx); err != nil { + panic(err) + } + } +} + +func setup(ctx context.Context, t testing.TB) (*mongo.Client, string) { + // TODO: Receive the URI from the environment variable + URI := "mongodb://root:root@mongodb:27017/" + client, err := OpenDB(ctx, Config{ + URI: URI, + }) + if err != nil { + panic(err) + } + + DBName := t.Name() + fmt.Sprintf("%d", time.Now().UnixNano()) + return client, DBName +} diff --git a/scrape_test.go b/scrape_test.go deleted file mode 100644 index 0c862c2..0000000 --- a/scrape_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package newsletter - -import ( - "fmt" - "testing" -) - -func TestFake(t *testing.T) { - fmt.Println("This test shouldn't fail") - t.Log("This test shouldn't fail") -}