From ff855c4990b4ddb2f14b664f2154351c9776bc6f Mon Sep 17 00:00:00 2001 From: Eli Yukelzon Date: Mon, 16 Dec 2024 17:41:04 +0100 Subject: [PATCH] added flashlight-tester, to be used by pinger (#1464) * added flashlight-tester, to be used by pinger --- .github/workflows/buid_tester.yml | 43 ++++++++ tester/Dockerfile | 22 ++++ tester/README.md | 48 ++++++++ tester/main.go | 176 ++++++++++++++++++++++++++++++ 4 files changed, 289 insertions(+) create mode 100644 .github/workflows/buid_tester.yml create mode 100644 tester/Dockerfile create mode 100644 tester/README.md create mode 100644 tester/main.go diff --git a/.github/workflows/buid_tester.yml b/.github/workflows/buid_tester.yml new file mode 100644 index 000000000..08ef450f3 --- /dev/null +++ b/.github/workflows/buid_tester.yml @@ -0,0 +1,43 @@ +name: Build and Push flashlight tester image +on: + push: + branches: [main] +jobs: + docker_push: + name: docker push + runs-on: + group: large-runners + permissions: + contents: "read" + id-token: "write" + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + with: + install: true + + - uses: google-github-actions/auth@v2 + id: google_auth + with: + token_format: access_token + workload_identity_provider: projects/472305719257/locations/global/workloadIdentityPools/github-actions/providers/ghactions-provider + service_account: ghactions@lantern-cloud.iam.gserviceaccount.com + + - name: docker login + uses: docker/login-action@v3 + with: + registry: us-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.google_auth.outputs.access_token }} + + - name: docker push + uses: docker/build-push-action@v5 + with: + context: . + push: true + cache-from: type=gha + cache-to: type=gha,mode=max + file: ./tester/Dockerfile + tags: | + us-docker.pkg.dev/lantern-cloud/containers/flashlight-tester:latest + us-docker.pkg.dev/lantern-cloud/containers/flashlight-tester:${{ github.sha }} diff --git a/tester/Dockerfile b/tester/Dockerfile new file mode 100644 index 000000000..977eebd72 --- /dev/null +++ b/tester/Dockerfile @@ -0,0 +1,22 @@ +FROM golang:1.22.6 AS builder + +RUN apt-get update && apt-get install -y git git-lfs bsdmainutils + +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /src + +COPY . . + +ENV GOCACHE=/root/.cache/go-build +RUN --mount=type=cache,target="$GOCACHE" GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /tester ./tester/... + +FROM debian:bookworm-slim + +RUN apt-get update && apt-get install -y ca-certificates curl && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /tester /tester + +ENV TRACE=true +ENTRYPOINT ["/tester"] diff --git a/tester/README.md b/tester/README.md new file mode 100644 index 000000000..14cd2e3c8 --- /dev/null +++ b/tester/README.md @@ -0,0 +1,48 @@ +# Flashlight tester + +This is a simple tester for the Flashlight library. +It receives all it's arguments via environment variables. + +> Note: this tester does not time out. It will run indefinitely until it's stopped, or it succeeds in pinging the target URL. + +## Environment variables + +- `DEVICE_ID`: The device id to use. +- `USER_ID`: The user id to use. +- `TOKEN`: The token to use. +- `RUN_ID`: The run id to use. It will be added to honeycomb traces as value of attribute `pinger-id`. (you can use it for looking up traces for the specific run) +- `TARGET_URL`: The target url to use. This is the url that will be pinged through flashlight. +- `DATA`: The path to use for config files and for logs. This is the path where the output will be written to (proxies.yaml, global.yaml, etc). You can place proxies.yaml there to use it instead of fetching. + +All of these are required. + +## CLI usage + +```bash +DEVICE_ID=1234 USER_ID=123 TOKEN=1234 RUN_ID=1234 TARGET_URL=https://example.com DATA=./mydir ./flashlight-tester +``` + +The tester will start flashlight, fetch the config & proxies and try to reach the target URL via the socks5 proxy that flashlight provides. +Upoon success, it will write the output of that request to the `output.txt`. + +## Docker usage + +On each new push to the repository, a new image of the tester is built and pushed to the registry. +It's tagged as `us-docker.pkg.dev/lantern-cloud/containers/flashlight-tester:FLASHLIGHT_HASH` + + +```bash +docker run --rm -v ./mydir:/output \ + -e DEVICE_ID=1234 \ + -e USER_ID=1234 \ + -e TOKEN=1234 \ + -e RUN_ID=1234 \ + -e TARGET_URL=https://example.com \ + -e DATA=/output \ + us-docker.pkg.dev/lantern-cloud/containers/flashlight-tester +``` + +## Passing custom proxies.yaml + +If you want to use a custom proxies.yaml, you can place it in the output directory and it will be used instead of fetching it from the server. +In order for flashlight to pick it up instead of using the fetched config, you need to specify another env variable: `STICKY=true` diff --git a/tester/main.go b/tester/main.go new file mode 100644 index 000000000..f24a3c385 --- /dev/null +++ b/tester/main.go @@ -0,0 +1,176 @@ +package main + +import ( + "fmt" + "io" + "os" + "os/exec" + "strconv" + "strings" + "sync" + "time" + + "github.com/google/uuid" + + "github.com/getlantern/flashlight/v7" + "github.com/getlantern/flashlight/v7/client" + "github.com/getlantern/flashlight/v7/common" + flashlightOtel "github.com/getlantern/flashlight/v7/otel" + "github.com/getlantern/flashlight/v7/stats" + "github.com/getlantern/golog" + "github.com/getlantern/ops" +) + +func configureOtel(country string) { + // Configure OpenTelemetry + const replacementText = "UUID-GOES-HERE" + const honeycombQueryTemplate = `https://ui.honeycomb.io/lantern-bc/environments/prod/datasets/flashlight?query=%7B%22time_range%22%3A300%2C%22granularity%22%3A15%2C%22breakdowns%22%3A%5B%5D%2C%22calculations%22%3A%5B%7B%22op%22%3A%22COUNT%22%7D%5D%2C%22filters%22%3A%5B%7B%22column%22%3A%22pinger-id%22%2C%22op%22%3A%22%3D%22%2C%22value%22%3A%22UUID-GOES-HERE%22%7D%5D%2C%22filter_combination%22%3A%22AND%22%2C%22orders%22%3A%5B%5D%2C%22havings%22%3A%5B%5D%2C%22trace_joins%22%3A%5B%5D%2C%22limit%22%3A100%7D` + runId := uuid.NewString() + fmt.Printf("performing lantern ping: url=%s\n", country) + fmt.Printf("lookup traces on Honeycomb with pinger-id: %s, link: %s\n", runId, strings.ReplaceAll(honeycombQueryTemplate, replacementText, runId)) + flashlightOtel.ConfigureOnce(&flashlightOtel.Config{ + Endpoint: "api.honeycomb.io:443", + Headers: map[string]string{ + "x-honeycomb-team": "vuWkzaeefr2OcL1SfowtuG", + }, + }, "pinger") + ops.SetGlobal("pinger-id", runId) +} + +func performLanternPing(urlToHit string, runId string, deviceId string, userId int64, token string, dataDir string, isSticky bool) error { + golog.SetPrepender(func(writer io.Writer) { + _, _ = writer.Write([]byte(fmt.Sprintf("%s: ", time.Now().Format("2006-01-02 15:04:05")))) + }) + + settings := common.NewUserConfigData("lantern", deviceId, userId, token, nil, "en-US") + statsTracker := stats.NewTracker() + var onOneProxy sync.Once + proxyReady := make(chan struct{}) + configureOtel(urlToHit) + common.LibraryVersion = "999.999.999" + fc, err := flashlight.New( + "pinger", + "999.999.999", + "10-10-2024", + dataDir, + false, + func() bool { return false }, + func() bool { return false }, + func() bool { return false }, + func() bool { return false }, + map[string]interface{}{ + "readableconfig": true, + "stickyconfig": isSticky, + }, + settings, + statsTracker, + func() bool { return false }, + func() string { return "en-US" }, + func(host string) (string, error) { + return host, nil + }, + func(category, action, label string) { + + }, + flashlight.WithOnDialError(func(err error, v bool) { + fmt.Printf("failed to dial %v %v\n", err, v) + }), + flashlight.WithOnSucceedingProxy(func() { + onOneProxy.Do(func() { + fmt.Printf("succeeding proxy\n") + proxyReady <- struct{}{} + }) + }), + ) + if err != nil { + return err + } + resultCh := make(chan error) + t1 := time.Now() + var t2, t3 time.Time + output := "" + go fc.Run("127.0.0.1:0", "127.0.0.1:0", func(cl *client.Client) { + go func() { + sa, ok := cl.Socks5Addr(5 * time.Second) + if !ok { + resultCh <- fmt.Errorf("failed to get socks5 address") + return + } + select { + case <-proxyReady: + break + } + + t2 = time.Now() + flashlightProxy := fmt.Sprintf("socks5://%s", sa) + fmt.Printf("lantern started correctly. urlToHit: %s flashlight proxy: %s\n", urlToHit, flashlightProxy) + + cmd := exec.Command("curl", "-x", flashlightProxy, "-s", urlToHit) + + // Run the command and capture the output + outputB, err := cmd.Output() + if err != nil { + fmt.Println("Error executing command:", err) + resultCh <- err + return + } + + output = string(outputB) + t3 = time.Now() + resultCh <- nil + }() + }, func(err error) { + resultCh <- err + }) + + var runErr error + select { + case err := <-resultCh: + runErr = err + break + } + defer fc.Stop() + + if runErr == nil { + fmt.Println("lantern ping completed successfully") + } + + _ = os.WriteFile(dataDir+"/output.txt", []byte(output), 0644) + + return os.WriteFile(dataDir+"/timing.txt", []byte(fmt.Sprintf(` +result: %v +run-id: %s +err: %v +started: %s +connected: %d +fetched: %d +url: %s`, + runErr == nil, runId, runErr, t1, int32(t2.Sub(t1).Milliseconds()), int32(t3.Sub(t1).Milliseconds()), urlToHit)), 0644) +} + +func main() { + deviceId := os.Getenv("DEVICE_ID") + userId := os.Getenv("USER_ID") + token := os.Getenv("TOKEN") + runId := os.Getenv("RUN_ID") + targetUrl := os.Getenv("TARGET_URL") + data := os.Getenv("DATA") + isSticky := os.Getenv("STICKY") == "true" + + if deviceId == "" || userId == "" || token == "" || runId == "" || targetUrl == "" || data == "" { + fmt.Println("missing required environment variable(s)") + fmt.Println("Required environment variables: DEVICE_ID, USER_ID, TOKEN, RUN_ID, TARGET_URL, DATA") + os.Exit(1) + } + + uid, err := strconv.ParseInt(userId, 10, 64) + if err != nil { + fmt.Println("failed to parse USER_ID") + os.Exit(1) + } + + if performLanternPing(targetUrl, runId, deviceId, uid, token, data, isSticky) != nil { + fmt.Println("failed to perform lantern ping") + os.Exit(1) + } +}