Merge pull request #1 from perebaj/initial_config

✨ init config

.github/workflows/ci.yaml

@@ -0,0 +1,19 @@
+name: CI-newsletter
+
+on: [push]
+
+jobs:
+  test:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Linting
+        run: make dev/lint
+
+      - name: Testing
+        run: make dev/test
+
+      - name: Container Image
+        run: make image

.golangci.yml

@@ -0,0 +1,11 @@
+linters:
+  enable:
+      - revive
+      - misspell
+      - gofmt
+issues:
+  # So we can detect exported/undocumented symbols
+  # https://github.com/golangci/golangci-lint/issues/2114
+  exclude:
+    - EXC0012
+  exclude-use-default: false

Dockerfile

@@ -0,0 +1,12 @@
+ARG GO_VERSION
+
+FROM docker.io/golang:${GO_VERSION}-alpine3.17 as builder
+RUN apk --no-cache add make bash
+WORKDIR /app  
+COPY . /app
+RUN make service
+
+FROM docker.io/alpine:3.17.2
+RUN apk --no-cache add ca-certificates
+COPY --from=builder /app/cmd/newsletter/newsletter /usr/bin/newsletter
+CMD ["newsletter"]

Dockerfile.dev

@@ -0,0 +1,20 @@
+ARG GO_VERSION
+
+FROM docker.io/golang:${GO_VERSION}-bullseye
+
+# Install necessary packages
+RUN apt-get update && \
+    apt-get install -y git
+
+ARG GOLANGCI_LINT_VERSION
+
+RUN go install github.com/golangci/golangci-lint/cmd/golangci-lint@${GOLANGCI_LINT_VERSION}
+
+RUN git config --global --add safe.directory /app/src
+
+WORKDIR /app/src
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .

Makefile

@@ -0,0 +1,98 @@
+# TOOLS VERSIONS
+export GO_VERSION=1.21.5
+export GOLANGCI_LINT_VERSION=v1.55.2
+devimage=newsletter-dev
+gopkg=$(devimage)-gopkg
+gocache=$(devimage)-gocache
+devrun=docker-compose run --rm newsletter
+image=perebaj
+version=$(shell git rev-parse --short HEAD)
+
+## run all tests. Usage `make test` or `make test testcase="TestFunctionName"` to run an isolated tests
+.PHONY: test
+test:
+	if [ -n "$(testcase)" ]; then \
+		go test ./... -timeout 10s -race -run="^$(testcase)$$" -v; \
+	else \
+		go test ./... -timeout 10s -race; \
+	fi
+
+## builds the service
+.PHONY: service
+service:
+	go build -o ./cmd/newsletter/newsletter ./cmd/newsletter
+
+## runs the service locally
+.PHONY: run
+run: service
+	./cmd/newsletter/newsletter
+
+## lint the whole project
+.PHONY: lint
+lint:
+	go run github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) run ./... 
+	go run golang.org/x/vuln/cmd/govulncheck@latest ./...
+
+
+## Build the service image
+.PHONY: image
+image:
+	docker build . \
+		--build-arg GO_VERSION=$(GO_VERSION) \
+		-t $(image)
+
+## Publish the service image
+.PHONY: image/publish
+image/publish: image
+	docker push $(image)
+
+.PHONY: dev
+dev: dev/image
+	$(devrun)
+
+## Create the dev container image
+.PHONY: dev/image
+dev/image:
+	docker build \
+		--build-arg GO_VERSION=$(GO_VERSION) \
+		--build-arg GOLANGCI_LINT_VERSION=$(GOLANGCI_LINT_VERSION) \
+		-t $(devimage) \
+		-f Dockerfile.dev \
+		.
+
+##run a make target inside the dev container
+dev/%: dev/image
+	$(devrun) make ${*}
+
+## Start containers, additionaly you can provide rebuild=true to force rebuild
+.PHONY: dev/start
+dev/start:
+	@echo "Starting development server..."
+	@if [ "$(rebuild)" = "true" ]; then \
+		docker-compose up -d --build; \
+	else \
+		docker-compose up -d; \
+	fi
+
+## Dev container logs
+.PHONY: dev/logs
+dev/logs:
+	docker-compose logs -f newsletter
+
+## Dev container stop
+.PHONY: dev/stop
+dev/stop:
+	docker-compose stop
+
+## Display help for all targets
+.PHONY: help
+help:
+	@awk '/^.PHONY: / { \
+		msg = match(lastLine, /^## /); \
+			if (msg) { \
+				cmd = substr($$0, 9, 100); \
+				msg = substr(lastLine, 4, 1000); \
+				printf "  ${GREEN}%-30s${RESET} %s\n", cmd, msg; \
+			} \
+	} \
+	{ lastLine = $$0 }' $(MAKEFILE_LIST)

cmd/newsletter/main.go

@@ -0,0 +1,32 @@
+// Package main is the initial point for the service newsletter.
+package main
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+// var references = []string{
+// 	"https://research.swtch.com/feed.atom",
+// 	"https://apenwarr.ca/sitemap.txt",
+// }
+
+func main() {
+
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://mongodb:27017"))
+	if err != nil {
+		panic(err)
+	}
+	err = client.Ping(ctx, nil)
+	if err != nil {
+		panic(err)
+	}
+	fmt.Println("PING")
+}

docker-compose.yml

@@ -0,0 +1,29 @@
+version: "3.8"
+
+services:
+  newsletter:
+    image: newsletter:dev
+    build:
+      context: .
+      dockerfile: ./Dockerfile.dev
+      args:
+        GO_VERSION: $GO_VERSION
+        GOLANGCI_LINT_VERSION: $GOLANGCI_LINT_VERSION
+    depends_on:
+      - mongodb
+    volumes:
+      - .:/app/src
+    command: go run ./cmd/newsletter
+
+  mongodb:
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: root
+      MONGO_INITDB_ROOT_PASSWORD: root
+    volumes:
+      - mongodbdata:/data/db
+    image: mongo:7.0
+
+volumes:
+  mongodbdata:
+
+

go.mod

@@ -0,0 +1,17 @@
+module github.com/perebaj/newsletter
+
+go 1.21.5
+
+require (
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.2 // indirect
+	github.com/xdg-go/stringprep v1.0.4 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	go.mongodb.org/mongo-driver v1.13.1 // indirect
+	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
+	golang.org/x/text v0.7.0 // indirect
+)

go.sum

@@ -0,0 +1,51 @@
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.mongodb.org/mongo-driver v1.13.1 h1:YIc7HTYsKndGK4RFzJ3covLz1byri52x0IoMB0Pt/vk=
+go.mongodb.org/mongo-driver v1.13.1/go.mod h1:wcDf1JBCXy2mOW0bWHwO/IOYqdca1MPCwDtFu/Z9+eo=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

scrape.go

@@ -0,0 +1,34 @@
+// Package newsletter is ----------------
+package newsletter
+
+import (
+	"bytes"
+	"net/http"
+)
+
+// GetReferences returns the content of the references
+func GetReferences(references []string) ([]string, error) {
+	refContent := make([]string, len(references))
+
+	for _, ref := range references {
+		resp, err := http.Get(ref)
+		if err != nil {
+			return nil, err
+		}
+
+		defer func() {
+			_ = resp.Body.Close()
+		}()
+
+		if resp.StatusCode == 200 {
+			buf := new(bytes.Buffer)
+			_, err := buf.ReadFrom(resp.Body)
+			if err != nil {
+				return nil, err
+			}
+			bodyString := buf.String()
+			refContent = append(refContent, bodyString[:100])
+		}
+	}
+	return refContent, nil
+}