initial commit - migration from private repository

.github/workflows/main.yml

@@ -0,0 +1,54 @@
+name: Check & test & build
+on: [push, pull_request]
+
+jobs:
+  check:
+    name: Quality & security checks
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.13
+
+      - name: Check out code
+        uses: actions/checkout@v1
+
+      - name: Check for secrets
+        uses: eshork/gitleaks-action@master
+
+      - name: Lint Go Code
+        run: |
+          export PATH=$PATH:$(go env GOPATH)/bin # temporary fix. See https://github.com/actions/setup-go/issues/14
+          make check
+
+  test:
+    name: Test & coverage
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.13
+
+      - name: Check out code
+        uses: actions/checkout@v1
+
+      - name: Run unit tests with
+        run: make test
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    needs: [check, test]
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.13
+
+      - name: Check out code
+        uses: actions/checkout@v1
+
+      - name: Build
+        run: make build

.github/workflows/release.yml

@@ -0,0 +1,57 @@
+name: Release
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  release:
+    name: Publish to github releases page
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.13
+
+      - name: Check out code
+        uses: actions/checkout@v1
+
+      - name: Release
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        run: |
+          export PATH=$PATH:$(go env GOPATH)/bin # temporary fix. See https://github.com/actions/setup-go/issues/14
+
+          VERSION=$(git describe --abbrev=0 --tags)
+          UNAME_SYS=$(uname -s)
+          UNAME_HW=$(uname -m)
+          TAR_THREATBITE=threatbite_${UNAME_SYS}_${UNAME_HW}.tar.gz
+          CHANGELOG=$(git log --oneline $(git describe --tags --abbrev=0 @^)..@)
+
+          make build
+          tar -cvzf ./bin/${TAR_THREATBITE} -C ./bin ./threatbite
+
+          go get github.com/tcnksm/ghr
+          ghr -t ${GITHUB_TOKEN} -b "${CHANGELOG}" -delete ${VERSION} ./bin/${TAR_THREATBITE}
+
+  registry:
+    name: Publish to docker hub
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+
+      - name: Version tag
+        uses: elgohr/Publish-Docker-Github-Action@master
+        with:
+          name: optimatiq/threatbite
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Latest tag
+        uses: elgohr/Publish-Docker-Github-Action@master
+        with:
+          name: optimatiq/threatbite
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+          tag_names: true

.gitignore

@@ -0,0 +1,26 @@
+# Binaries for programs and plugins
+bin/
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+*.out
+
+# Test binary, build with `go test -c`
+*.test
+
+# GoLand
+.idea/
+
+# vs-code
+.vscode/
+
+# vendor
+vendor/
+
+# local config file
+config_local.env
+
+# generated dynamically
+resources/maxmind/

Dockerfile

@@ -0,0 +1,22 @@
+FROM golang:1.13 AS builder
+
+WORKDIR /app
+
+COPY go.mod .
+COPY go.sum .
+RUN go mod download
+
+COPY k8s/reputator .
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 make build
+
+FROM alpine:latest
+
+RUN apk --no-cache add ca-certificates
+WORKDIR /root/
+COPY --from=builder /app/bin .
+COPY --from=builder /app/resources ./resources/
+
+ENV PORT 8080
+ENV DEBUG 1
+
+CMD ["./reputator"]

Makefile

@@ -0,0 +1,67 @@
+SHELL := /bin/bash
+export GO111MODULE=on
+export GOPROXY=https://proxy.golang.org
+
+.DEFAULT_GOAL: all
+
+GIT_TAG := `git describe --abbrev=0 --tags`
+GIT_COMMIT := `git rev-parse HEAD`
+
+LDFLAGS=-ldflags "-s -w -X=main.date=$(shell date +%FT%T%z) -X=main.tag=$(GIT_TAG) -X=main.commit=$(GIT_COMMIT) "
+
+.PHONY: build check clean format format-check git-tag-major git-tag-minor git-tag-patch help test tidy
+
+all: check test build ## Default target: check, test, build,
+
+build: ## Build all excecutables, located under ./bin/
+	@echo "[threatbite] Building..."
+	@go build -trimpath -o ./bin/threatbite $(LDFLAGS) cmd/threatbite/main.go
+
+clean: ## Remove all artifacts from ./bin/ and ./resources
+	@rm -rf ./bin/*
+
+format: ## Format go code with goimports
+	@go get golang.org/x/tools/cmd/goimports
+	@goimports -l -w .
+
+format-check: ## Check if the code is formatted
+	@go get golang.org/x/tools/cmd/goimports
+	@for i in $$(goimports -l .); do echo "[ERROR] Code is not formated run 'make format'" && exit 1; done
+
+test: ## Run tests
+	@go test -race  ./...
+
+tidy: ## Run go mod tidy
+	@go mod tidy
+
+check: format-check ## Linting and static analysis
+	@if grep -r --include='*.go' -E "fmt.Print|spew.Dump" *; then \
+		echo "code contains fmt.Print* or spew.Dump function"; \
+		exit 1; \
+	fi
+
+	@if test ! -e ./bin/golangci-lint; then \
+		curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh; \
+	fi
+	@./bin/golangci-lint run --timeout 180s -E gosec -E stylecheck -E golint -E goimports -E whitespace
+
+git-tag-patch: ## Push new tag to repository with patch number incremented
+	$(eval NEW_VERSION=$(shell git describe --tags --abbrev=0 | awk -F'[a-z.]' '{$$4++;print "v" $$2 "." $$3 "." $$4}'))
+	@echo Version: $(NEW_VERSION)
+	@git tag -a $(NEW_VERSION) -m "new patch release"
+	@git push origin $(NEW_VERSION)
+
+git-tag-minor: ## Push new tag to repository with minor number incremented
+	$(eval NEW_VERSION=$(shell git describe --tags --abbrev=0 | awk -F'[a-z.]' '{$$3++;print "v" $$2 "." $$3 "." 0}'))
+	@echo Version: $(NEW_VERSION)
+	@git tag -a $(NEW_VERSION) -m "new minor release"
+	@git push origin $(NEW_VERSION)
+
+git-tag-major:  ## Push new tag to repository with major number incremented
+	$(eval NEW_VERSION=$(shell git describe --tags --abbrev=0 | awk -F'[a-z.]' '{$$2++;print "v" $$2 "." 0 "." 0}'))
+	@echo Version: $(NEW_VERSION)
+	@git tag -a $(NEW_VERSION) -m "new major release"
+	@git push origin $(NEW_VERSION)
+
+help: ## Show help
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'

README.md

@@ -0,0 +1,159 @@
+# IP Reputation tool
+
+![GithubActions](https://github.com/optimatiq/threatbite/workflows/Check%20&%20test%20&%20build/badge.svg)
+
+# About
+ThreatBite is a real-time service that detects unwanted web users. It takes into account IP addresses, e-mails or HTTP request headers.
+
+# Features
+
+### Identifying the source of threat
+ThreatBite identifies potential sources of fraud by comparing user identification data to over 500 databases of bad internet actors.
+
+### Account creation protection
+ThreatBite protects against automatic account creation and user account hijacking.
+
+### Spam detection
+ThreatBite Identifies potential sources of spammers.
+
+### Tor users detection
+ThreatBite recognizes addresses belonging to the Tor network.
+
+### Proxy/VPN
+ThreatBite detects addresses that are used as proxys or VPNs.
+
+## Download
+- Grab the latest binary from the [releases](https://github.com/optimatiq/threatbite/releases) page and run it:
+
+```shell
+./threatbite
+```
+- Or use the official Docker image:
+
+```shell
+docker run -d -p 8080:8080 optimatiq/threatbite
+```
+
+- Or get the sources:
+
+```shell
+git clone https://github.com/optimatiq/threatbite
+cd ./threatbite
+make bulid && ./bin/threatbite
+```
+
+## Quickstart
+
+### Scoring for email
+`curl localhost:8080/v1/score/email/[email protected]`
+
+### Scoring for IP address
+`curl  localhost:8080/v1/score/ip/1.1.1.1`
+
+### Scoring for HTTP request
+
+```
+curl \
+  -X POST \
+  localhost:8080/v1/score/request \
+  -H 'Content-Type: application/json' \
+  -d '{"ip":"1.2.3.4", "host":"host.pl", "uri":"/", "method":"GET", "user_agent":"curl", "headers": {"x-header": 1}}'
+```
+
+or 
+
+```
+curl \
+  -X POST \
+  localhost:8080/v1/score/request \
+  -d 'ip=1.2.3.4' \
+  -d 'host=host.pl' \
+  -d 'uri=/' \
+  -d 'method=POST' \
+  -d 'user_agent=curl'
+```
+### API documentation
+`chrome localhost:8080`
+
+### Rate limits
+10 requests per seconds are allowed, after reaching limit 429 HTTP status code is returned
+
+### Configuration
+Configuration is done via env variables or config.env file. All parameters are optional:
+* `PORT`       - API listening port default 8080
+* `DEBUG`      - values: false, true, 1, 0 or empty
+* `AUTO_TLS`   - values: false, true, 1, 0 or empty, automatic access to certificates from Let's Encrypt
+
+License keys for these external services will improve the quality of the results. It is highly recommended to set them.
+* `PWNED_KEY`   - obtained from https://haveibeenpwned.com/
+* `MAXMIND_KEY` - obtained from https://www.maxmind.com/en/accounts/current/license-key   
+
+# TODO 
+* `SMTP_HELLO` - # TODO  
+* `SMTP_FROM`  - # TODO 
+
+IP/CIDR lists contain information about addresses used as proxy/VPN or other malicious activity. 
+You can provide one or many sources separated by whitespace. 
+The format of the data is straightforward, and each line contains one IP or CIDR addresses.
+Threadbite open-source version provides public sources that are limited in scope and might be outdated with no SLA. 
+If you interested in curated and more accurate lists with SLA, please contact us at [email protected]
+
+* `PROXY_LIST` - URL or set of URLs separated by space, default: https://get.threatbite.com/public/proxy.txt
+* `SPAM_LIST`  - URL or set of URLs separated by space, default: https://get.threatbite.com/public/spam.txt
+* `VPN_LIST`   - URL or set of URLs separated by space, default: https://get.threatbite.com/public/vpn.txt
+* `DC_LIST`    - URL or set of URLs separated by space, default: https://get.threatbite.com/public/dc-names.txt
+
+Email lists contain information about domains used as disposal emails or free solutions which are often used in spam or phishing campaigns.
+You can provide one or many sources separated by whitespace. 
+The format of the data is straightforward, and each line contains one domain
+Threadbite open-source version provides public sources that are limited in scope and might be outdated with no SLA. 
+If you interested in curated and more accurate lists with SLA, please contact us at [email protected]
+
+* `EMAIL_DISPOSAL_LIST`  - URL or set of URLs separated by space, which point to the IP/Net source of data
+* `EMAIL_FREE_LIST    `  - URL or set of URLs separated by space, which point to the IP/Net source of data
+
+### config.env file 
+You can store your custom configuration in config.env. The format is defined as below:
+
+```
+DEBUG=true
+PORT=443
+AUTO_TLS=true
+PROXY_LIST=https://provider1.com https://provider2.com
+```
+
+By default threatbite binary is looking for config.env file in the same directory, 
+but you can use `-config` flag to change this and point to any file in a filesystem.
+
+`./bin/threatbite -confg=/etc/threatbite/config.env`
+
+## Development
+
+### Go 
+At least version 1.13 is required 
+
+### Building & running
+`make bulid && ./bin/threatbite`
+
+### Run tests:
+`make test`
+
+### Quality & linteners:
+`make check`
+
+### Other targets
+`make help`
+
+### Internal endpoints
+`/internal/*` endpoints should not be public, they contains sensitive data. 
+
+### Health check
+`/internal/health`
+
+### Monitoring
+Prometheus endpoint is available at: `/internal/metrics`
+
+### Profiling
+`go tool pprof localhost:8080/internal/debug/pprof/profile?seconds=20`
+
+`go tool pprof localhost:8080/internal/debug/pprof/heap`