diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..5ea8a94 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,19 @@ +name: CI-newsletter + +on: [push] + +jobs: + test: + runs-on: ubuntu-22.04 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Linting + run: make dev/lint + + - name: Testing + run: make dev/test + + - name: Container Image + run: make image diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..254224d --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,11 @@ +linters: + enable: + - revive + - misspell + - gofmt +issues: + # So we can detect exported/undocumented symbols + # https://github.com/golangci/golangci-lint/issues/2114 + exclude: + - EXC0012 + exclude-use-default: false diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a5d16f6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +ARG GO_VERSION + +FROM docker.io/golang:${GO_VERSION}-alpine3.17 as builder +RUN apk --no-cache add make bash +WORKDIR /app +COPY . /app +RUN make service + +FROM docker.io/alpine:3.17.2 +RUN apk --no-cache add ca-certificates +COPY --from=builder /app/cmd/newsletter/newsletter /usr/bin/newsletter +CMD ["newsletter"] diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 0000000..bc9b428 --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,20 @@ +ARG GO_VERSION + +FROM docker.io/golang:${GO_VERSION}-bullseye + +# Install necessary packages +RUN apt-get update && \ + apt-get install -y git + +ARG GOLANGCI_LINT_VERSION + +RUN go install github.com/golangci/golangci-lint/cmd/golangci-lint@${GOLANGCI_LINT_VERSION} + +RUN git config --global --add safe.directory /app/src + +WORKDIR /app/src + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..cf35dc1 --- /dev/null +++ b/Makefile @@ -0,0 +1,98 @@ +# TOOLS VERSIONS +export GO_VERSION=1.21.5 +export GOLANGCI_LINT_VERSION=v1.55.2 +devimage=newsletter-dev +gopkg=$(devimage)-gopkg +gocache=$(devimage)-gocache +devrun=docker-compose run --rm newsletter +image=perebaj +version=$(shell git rev-parse --short HEAD) + +## run all tests. Usage `make test` or `make test testcase="TestFunctionName"` to run an isolated tests +.PHONY: test +test: + if [ -n "$(testcase)" ]; then \ + go test ./... -timeout 10s -race -run="^$(testcase)$$" -v; \ + else \ + go test ./... -timeout 10s -race; \ + fi + +## builds the service +.PHONY: service +service: + go build -o ./cmd/newsletter/newsletter ./cmd/newsletter + +## runs the service locally +.PHONY: run +run: service + ./cmd/newsletter/newsletter + +## lint the whole project +.PHONY: lint +lint: + go run github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) run ./... + go run golang.org/x/vuln/cmd/govulncheck@latest ./... + + +## Build the service image +.PHONY: image +image: + docker build . \ + --build-arg GO_VERSION=$(GO_VERSION) \ + -t $(image) + +## Publish the service image +.PHONY: image/publish +image/publish: image + docker push $(image) + +.PHONY: dev +dev: dev/image + $(devrun) + +## Create the dev container image +.PHONY: dev/image +dev/image: + docker build \ + --build-arg GO_VERSION=$(GO_VERSION) \ + --build-arg GOLANGCI_LINT_VERSION=$(GOLANGCI_LINT_VERSION) \ + -t $(devimage) \ + -f Dockerfile.dev \ + . + +##run a make target inside the dev container +dev/%: dev/image + $(devrun) make ${*} + +## Start containers, additionaly you can provide rebuild=true to force rebuild +.PHONY: dev/start +dev/start: + @echo "Starting development server..." + @if [ "$(rebuild)" = "true" ]; then \ + docker-compose up -d --build; \ + else \ + docker-compose up -d; \ + fi + +## Dev container logs +.PHONY: dev/logs +dev/logs: + docker-compose logs -f newsletter + +## Dev container stop +.PHONY: dev/stop +dev/stop: + docker-compose stop + +## Display help for all targets +.PHONY: help +help: + @awk '/^.PHONY: / { \ + msg = match(lastLine, /^## /); \ + if (msg) { \ + cmd = substr($$0, 9, 100); \ + msg = substr(lastLine, 4, 1000); \ + printf " ${GREEN}%-30s${RESET} %s\n", cmd, msg; \ + } \ + } \ + { lastLine = $$0 }' $(MAKEFILE_LIST) diff --git a/cmd/newsletter/main.go b/cmd/newsletter/main.go new file mode 100644 index 0000000..6ec3943 --- /dev/null +++ b/cmd/newsletter/main.go @@ -0,0 +1,32 @@ +// Package main is the initial point for the service newsletter. +package main + +import ( + "context" + "fmt" + "time" + + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// var references = []string{ +// "https://research.swtch.com/feed.atom", +// "https://apenwarr.ca/sitemap.txt", +// } + +func main() { + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://mongodb:27017")) + if err != nil { + panic(err) + } + err = client.Ping(ctx, nil) + if err != nil { + panic(err) + } + fmt.Println("PING") +} diff --git a/cmd/newsletter/newsletter b/cmd/newsletter/newsletter new file mode 100755 index 0000000..3f94e3b Binary files /dev/null and b/cmd/newsletter/newsletter differ diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..2c9d7c0 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,29 @@ +version: "3.8" + +services: + newsletter: + image: newsletter:dev + build: + context: . + dockerfile: ./Dockerfile.dev + args: + GO_VERSION: $GO_VERSION + GOLANGCI_LINT_VERSION: $GOLANGCI_LINT_VERSION + depends_on: + - mongodb + volumes: + - .:/app/src + command: go run ./cmd/newsletter + + mongodb: + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: root + volumes: + - mongodbdata:/data/db + image: mongo:7.0 + +volumes: + mongodbdata: + + diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..07138c2 --- /dev/null +++ b/go.mod @@ -0,0 +1,17 @@ +module github.com/perebaj/newsletter + +go 1.21.5 + +require ( + github.com/golang/snappy v0.0.1 // indirect + github.com/klauspost/compress v1.13.6 // indirect + github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect + github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + go.mongodb.org/mongo-driver v1.13.1 // indirect + golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect + golang.org/x/text v0.7.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..052dd3c --- /dev/null +++ b/go.sum @@ -0,0 +1,51 @@ +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.mongodb.org/mongo-driver v1.13.1 h1:YIc7HTYsKndGK4RFzJ3covLz1byri52x0IoMB0Pt/vk= +go.mongodb.org/mongo-driver v1.13.1/go.mod h1:wcDf1JBCXy2mOW0bWHwO/IOYqdca1MPCwDtFu/Z9+eo= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/scrape.go b/scrape.go new file mode 100644 index 0000000..6ac8137 --- /dev/null +++ b/scrape.go @@ -0,0 +1,34 @@ +// Package newsletter is ---------------- +package newsletter + +import ( + "bytes" + "net/http" +) + +// GetReferences returns the content of the references +func GetReferences(references []string) ([]string, error) { + refContent := make([]string, len(references)) + + for _, ref := range references { + resp, err := http.Get(ref) + if err != nil { + return nil, err + } + + defer func() { + _ = resp.Body.Close() + }() + + if resp.StatusCode == 200 { + buf := new(bytes.Buffer) + _, err := buf.ReadFrom(resp.Body) + if err != nil { + return nil, err + } + bodyString := buf.String() + refContent = append(refContent, bodyString[:100]) + } + } + return refContent, nil +}