added flashlight-tester, to be used by pinger (#1464)

* added flashlight-tester, to be used by pinger

.github/workflows/buid_tester.yml

@@ -0,0 +1,43 @@
+name: Build and Push flashlight tester image
+on:
+  push:
+    branches: [main]
+jobs:
+  docker_push:
+    name: docker push
+    runs-on:
+      group: large-runners
+    permissions:
+      contents: "read"
+      id-token: "write"
+    steps:
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
+        with:
+          install: true
+
+      - uses: google-github-actions/auth@v2
+        id: google_auth
+        with:
+          token_format: access_token
+          workload_identity_provider: projects/472305719257/locations/global/workloadIdentityPools/github-actions/providers/ghactions-provider
+          service_account: [email protected]
+
+      - name: docker login
+        uses: docker/login-action@v3
+        with:
+          registry: us-docker.pkg.dev
+          username: oauth2accesstoken
+          password: ${{ steps.google_auth.outputs.access_token }}
+
+      - name: docker push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          file: ./tester/Dockerfile
+          tags: |
+            us-docker.pkg.dev/lantern-cloud/containers/flashlight-tester:latest
+            us-docker.pkg.dev/lantern-cloud/containers/flashlight-tester:${{ github.sha }}

tester/Dockerfile

@@ -0,0 +1,22 @@
+FROM golang:1.22.6 AS builder
+
+RUN apt-get update && apt-get install -y git git-lfs bsdmainutils
+
+ARG TARGETOS
+ARG TARGETARCH
+
+WORKDIR /src
+
+COPY . .
+
+ENV GOCACHE=/root/.cache/go-build
+RUN --mount=type=cache,target="$GOCACHE" GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /tester ./tester/...
+
+FROM debian:bookworm-slim
+
+RUN apt-get update && apt-get install -y ca-certificates curl && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /tester /tester
+
+ENV TRACE=true
+ENTRYPOINT ["/tester"]

tester/README.md

@@ -0,0 +1,48 @@
+# Flashlight tester
+
+This is a simple tester for the Flashlight library.
+It receives all it's arguments via environment variables.
+
+> Note: this tester does not time out. It will run indefinitely until it's stopped, or it succeeds in pinging the target URL.
+
+## Environment variables
+
+- `DEVICE_ID`: The device id to use.
+- `USER_ID`: The user id to use.
+- `TOKEN`: The token to use.
+- `RUN_ID`: The run id to use. It will be added to honeycomb traces as value of attribute `pinger-id`. (you can use it for looking up traces for the specific run)
+- `TARGET_URL`: The target url to use. This is the url that will be pinged through flashlight.
+- `DATA`: The path to use for config files and for logs. This is the path where the output will be written to (proxies.yaml, global.yaml, etc). You can place proxies.yaml there to use it instead of fetching.
+
+All of these are required.
+
+## CLI usage
+
+```bash
+DEVICE_ID=1234 USER_ID=123 TOKEN=1234 RUN_ID=1234 TARGET_URL=https://example.com DATA=./mydir ./flashlight-tester
+```
+
+The tester will start flashlight, fetch the config & proxies and try to reach the target URL via the socks5 proxy that flashlight provides.
+Upoon success, it will write the output of that request to the `output.txt`.
+
+## Docker usage
+
+On each new push to the repository, a new image of the tester is built and pushed to the registry.
+It's tagged as `us-docker.pkg.dev/lantern-cloud/containers/flashlight-tester:FLASHLIGHT_HASH`
+
+
+```bash
+docker run --rm -v ./mydir:/output \
+    -e DEVICE_ID=1234 \
+    -e USER_ID=1234 \
+    -e TOKEN=1234 \
+    -e RUN_ID=1234 \
+    -e TARGET_URL=https://example.com \
+    -e DATA=/output \
+   us-docker.pkg.dev/lantern-cloud/containers/flashlight-tester
+```
+
+## Passing custom proxies.yaml
+
+If you want to use a custom proxies.yaml, you can place it in the output directory and it will be used instead of fetching it from the server.
+In order for flashlight to pick it up instead of using the fetched config, you need to specify another env variable: `STICKY=true`

tester/main.go

@@ -0,0 +1,176 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/google/uuid"
+
+	"github.com/getlantern/flashlight/v7"
+	"github.com/getlantern/flashlight/v7/client"
+	"github.com/getlantern/flashlight/v7/common"
+	flashlightOtel "github.com/getlantern/flashlight/v7/otel"
+	"github.com/getlantern/flashlight/v7/stats"
+	"github.com/getlantern/golog"
+	"github.com/getlantern/ops"
+)
+
+func configureOtel(country string) {
+	// Configure OpenTelemetry
+	const replacementText = "UUID-GOES-HERE"
+	const honeycombQueryTemplate = `https://ui.honeycomb.io/lantern-bc/environments/prod/datasets/flashlight?query=%7B%22time_range%22%3A300%2C%22granularity%22%3A15%2C%22breakdowns%22%3A%5B%5D%2C%22calculations%22%3A%5B%7B%22op%22%3A%22COUNT%22%7D%5D%2C%22filters%22%3A%5B%7B%22column%22%3A%22pinger-id%22%2C%22op%22%3A%22%3D%22%2C%22value%22%3A%22UUID-GOES-HERE%22%7D%5D%2C%22filter_combination%22%3A%22AND%22%2C%22orders%22%3A%5B%5D%2C%22havings%22%3A%5B%5D%2C%22trace_joins%22%3A%5B%5D%2C%22limit%22%3A100%7D`
+	runId := uuid.NewString()
+	fmt.Printf("performing lantern ping: url=%s\n", country)
+	fmt.Printf("lookup traces on Honeycomb with pinger-id: %s, link: %s\n", runId, strings.ReplaceAll(honeycombQueryTemplate, replacementText, runId))
+	flashlightOtel.ConfigureOnce(&flashlightOtel.Config{
+		Endpoint: "api.honeycomb.io:443",
+		Headers: map[string]string{
+			"x-honeycomb-team": "vuWkzaeefr2OcL1SfowtuG",
+		},
+	}, "pinger")
+	ops.SetGlobal("pinger-id", runId)
+}
+
+func performLanternPing(urlToHit string, runId string, deviceId string, userId int64, token string, dataDir string, isSticky bool) error {
+	golog.SetPrepender(func(writer io.Writer) {
+		_, _ = writer.Write([]byte(fmt.Sprintf("%s: ", time.Now().Format("2006-01-02 15:04:05"))))
+	})
+
+	settings := common.NewUserConfigData("lantern", deviceId, userId, token, nil, "en-US")
+	statsTracker := stats.NewTracker()
+	var onOneProxy sync.Once
+	proxyReady := make(chan struct{})
+	configureOtel(urlToHit)
+	common.LibraryVersion = "999.999.999"
+	fc, err := flashlight.New(
+		"pinger",
+		"999.999.999",
+		"10-10-2024",
+		dataDir,
+		false,
+		func() bool { return false },
+		func() bool { return false },
+		func() bool { return false },
+		func() bool { return false },
+		map[string]interface{}{
+			"readableconfig": true,
+			"stickyconfig":   isSticky,
+		},
+		settings,
+		statsTracker,
+		func() bool { return false },
+		func() string { return "en-US" },
+		func(host string) (string, error) {
+			return host, nil
+		},
+		func(category, action, label string) {
+
+		},
+		flashlight.WithOnDialError(func(err error, v bool) {
+			fmt.Printf("failed to dial %v %v\n", err, v)
+		}),
+		flashlight.WithOnSucceedingProxy(func() {
+			onOneProxy.Do(func() {
+				fmt.Printf("succeeding proxy\n")
+				proxyReady <- struct{}{}
+			})
+		}),
+	)
+	if err != nil {
+		return err
+	}
+	resultCh := make(chan error)
+	t1 := time.Now()
+	var t2, t3 time.Time
+	output := ""
+	go fc.Run("127.0.0.1:0", "127.0.0.1:0", func(cl *client.Client) {
+		go func() {
+			sa, ok := cl.Socks5Addr(5 * time.Second)
+			if !ok {
+				resultCh <- fmt.Errorf("failed to get socks5 address")
+				return
+			}
+			select {
+			case <-proxyReady:
+				break
+			}
+
+			t2 = time.Now()
+			flashlightProxy := fmt.Sprintf("socks5://%s", sa)
+			fmt.Printf("lantern started correctly. urlToHit: %s flashlight proxy: %s\n", urlToHit, flashlightProxy)
+
+			cmd := exec.Command("curl", "-x", flashlightProxy, "-s", urlToHit)
+
+			// Run the command and capture the output
+			outputB, err := cmd.Output()
+			if err != nil {
+				fmt.Println("Error executing command:", err)
+				resultCh <- err
+				return
+			}
+
+			output = string(outputB)
+			t3 = time.Now()
+			resultCh <- nil
+		}()
+	}, func(err error) {
+		resultCh <- err
+	})
+
+	var runErr error
+	select {
+	case err := <-resultCh:
+		runErr = err
+		break
+	}
+	defer fc.Stop()
+
+	if runErr == nil {
+		fmt.Println("lantern ping completed successfully")
+	}
+
+	_ = os.WriteFile(dataDir+"/output.txt", []byte(output), 0644)
+
+	return os.WriteFile(dataDir+"/timing.txt", []byte(fmt.Sprintf(`
+result: %v
+run-id: %s
+err: %v
+started: %s
+connected: %d
+fetched: %d
+url: %s`,
+		runErr == nil, runId, runErr, t1, int32(t2.Sub(t1).Milliseconds()), int32(t3.Sub(t1).Milliseconds()), urlToHit)), 0644)
+}
+
+func main() {
+	deviceId := os.Getenv("DEVICE_ID")
+	userId := os.Getenv("USER_ID")
+	token := os.Getenv("TOKEN")
+	runId := os.Getenv("RUN_ID")
+	targetUrl := os.Getenv("TARGET_URL")
+	data := os.Getenv("DATA")
+	isSticky := os.Getenv("STICKY") == "true"
+
+	if deviceId == "" || userId == "" || token == "" || runId == "" || targetUrl == "" || data == "" {
+		fmt.Println("missing required environment variable(s)")
+		fmt.Println("Required environment variables: DEVICE_ID, USER_ID, TOKEN, RUN_ID, TARGET_URL, DATA")
+		os.Exit(1)
+	}
+
+	uid, err := strconv.ParseInt(userId, 10, 64)
+	if err != nil {
+		fmt.Println("failed to parse USER_ID")
+		os.Exit(1)
+	}
+
+	if performLanternPing(targetUrl, runId, deviceId, uid, token, data, isSticky) != nil {
+		fmt.Println("failed to perform lantern ping")
+		os.Exit(1)
+	}
+}