add junit report + buildkite upload (#68)

Fixes #65

and re-arranged some stuff:
* split out all report renders to their own file
* update result struct to track per-test runtime
* disable failing web5-kt test
* use web5-js test from web5-js repo, drop web5-js test from this repo
* renamed the go package to reflect the current repo name

.github/workflows/pages.yaml

@@ -11,7 +11,11 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
       - name: run for all SDKs
-        run: cd test-harness && go run ./cmd/web5-test-harness many sdks/* && mv _site ../_site
+        run: |
+          git clone https://github.com/TBD54566975/web5-js.git
+          git clone https://github.com/TBD54566975/web5-kt.git
+          cd test-harness
+          go run ./cmd/web5-test-harness many ../web5-* && mv _site ../_site
       - name: Copy test-vectors
         run: cp -r ./web5-test-vectors _site/
       - uses: actions/upload-pages-artifact@v2

.github/workflows/self-test.yaml

@@ -10,11 +10,15 @@ jobs:
       matrix:
         SDK:
           - web5-js
-          - web5-kt
+          # - web5-kt
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
       - name: self test
-        run: cd test-harness && go run ./cmd/web5-test-harness one sdks/$SDK
+        run: |
+          git clone https://github.com/TBD54566975/${SDK} sdk
+          cd test-harness
+          go run ./cmd/web5-test-harness one ../sdk
         env:
           SDK: ${{ matrix.SDK }}
+          BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }}

CONTRIBUTING.md

@@ -5,7 +5,7 @@ There are many ways to be an open source contributor, and we're here to help you
 
 * Propose ideas in our
   [discord](https://discord.gg/tbd)
-* Raise an issue or feature request in our [issue tracker](https://github.com/TBD54566975/web5-spec/issues)
+* Raise an issue or feature request in our [issue tracker](https://github.com/TBD54566975/sdk-development/issues)
 * Help another contributor with one of their questions, or a code review
 * Suggest improvements to our Getting Started documentation by supplying a Pull Request
 * Evangelize our work together in conferences, podcasts, and social media spaces.
@@ -171,7 +171,7 @@ $> ./gradlew clean build test
 ### Issues
 
 Anyone from the community is welcome (and encouraged!) to raise issues via
-[GitHub Issues](https://github.com/TBD54566975/web5-spec/issues).
+[GitHub Issues](https://github.com/TBD54566975/sdk-development/issues).
 
 ### Discussions
 
@@ -181,7 +181,7 @@ We advocate an asynchronous, written debate model - so write up your thoughts an
 
 ### Continuous Integration
 
-Build and Test cycles are run on every commit to every branch on [GitHub Actions](https://github.com/TBD54566975/web5-spec/actions).
+Build and Test cycles are run on every commit to every branch on [GitHub Actions](https://github.com/TBD54566975/sdk-development/actions).
 
 ## Contribution
 

test-harness/cmd/web5-test-harness/test-many.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/TBD54566975/web5-spec/reports"
+	"github.com/TBD54566975/sdk-development/reports"
 	"github.com/spf13/cobra"
 	"golang.org/x/exp/slog"
 )

test-harness/cmd/web5-test-harness/test-one.go

@@ -1,15 +1,20 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
-	"github.com/TBD54566975/web5-spec/openapi"
-	"github.com/TBD54566975/web5-spec/reports"
-	"github.com/TBD54566975/web5-spec/tests"
+	"github.com/TBD54566975/sdk-development/openapi"
+	"github.com/TBD54566975/sdk-development/reports"
+	"github.com/TBD54566975/sdk-development/tests"
 	"github.com/spf13/cobra"
 	"golang.org/x/exp/slog"
 )
@@ -43,6 +48,20 @@ var (
 				}
 			}
 
+			buildkiteToken := os.Getenv("BUILDKITE_ANALYTICS_TOKEN")
+			if buildkiteToken != "" {
+				if err := buildkiteUpload(report, buildkiteToken); err != nil {
+					slog.Error("error sending results to buildkite", "error", err)
+				}
+			}
+
+			junitFile := os.Getenv("JUNIT_REPORT")
+			if junitFile != "" {
+				if err := reports.WriteJunitToFile(report, junitFile); err != nil {
+					slog.Error("error writing junit report", "file", junitFile, "error", err)
+				}
+			}
+
 			if !report.IsPassing() {
 				os.Exit(1)
 			}
@@ -129,3 +148,88 @@ func testOne(dir string) (reports.Report, error) {
 func init() {
 	root.AddCommand(testOneCmd)
 }
+
+func buildkiteUpload(report reports.Report, buildkiteToken string) error {
+	slog.Info("uploading junit report to buildkit")
+
+	var body bytes.Buffer
+	writer := multipart.NewWriter(&body)
+
+	formFields := map[string]string{
+		"format":              "junit",
+		"run_env[CI]":         "github_actions",
+		"run_env[key]":        fmt.Sprintf("%s-%s-%s", os.Getenv("GITHUB_ACTION"), os.Getenv("GITHUB_RUN_NUMBER"), os.Getenv("GITHUB_RUN_ATTEMPT")),
+		"run_env[number]":     os.Getenv("GITHUB_RUN_NUMBER"),
+		"run_env[branch]":     os.Getenv("GITHUB_REF"),
+		"run_env[commit_sha]": os.Getenv("GITHUB_SHA"),
+		"run_env[url]":        fmt.Sprintf("https://github.com/%s/actions/runs/%s", os.Getenv("GITHUB_REPOSITORY"), os.Getenv("GITHUB_RUN_ID")),
+	}
+	for k, v := range formFields {
+		if err := addFormValue(writer, k, v); err != nil {
+			return err
+		}
+	}
+
+	part, err := writer.CreateFormFile("data", "web5-test-harness.xml")
+	if err != nil {
+		slog.Error("error creating form file")
+		return err
+	}
+
+	if err := reports.WriteJunit(report, part); err != nil {
+		slog.Error("error generating junit report")
+		return err
+	}
+
+	if err := writer.Close(); err != nil {
+		slog.Error("error closing multi-part form writer")
+		return err
+	}
+
+	req, err := http.NewRequest(http.MethodPost, "https://analytics-api.buildkite.com/v1/uploads", &body)
+	if err != nil {
+		slog.Error("error constructing request to buildkite")
+		return err
+	}
+
+	req.Header.Add("Content-Type", writer.FormDataContentType())
+	req.Header.Add("Authorization", fmt.Sprintf("Token token=%s", buildkiteToken))
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		slog.Error("error uploading results to buildkite")
+		return err
+	}
+	defer resp.Body.Close()
+
+	responseBody := bytes.Buffer{}
+	if _, err := io.Copy(&responseBody, resp.Body); err != nil {
+		slog.Error("error reading response from buildkite")
+		return err
+	}
+
+	if resp.StatusCode > 299 {
+		slog.Error("unexpected response status from buildkite", "status", resp.Status, "response", body.String())
+		return fmt.Errorf("unexpected %s", resp.Status)
+	}
+
+	slog.Info("successfully uploaded results to buildkite", "response", body.String())
+
+	return nil
+}
+
+func addFormValue(writer *multipart.Writer, key, value string) error {
+	field, err := writer.CreateFormField(key)
+	if err != nil {
+		slog.Error("error creating form field", "key", key, "value", value)
+		return err
+	}
+
+	_, err = io.Copy(field, strings.NewReader(value))
+	if err != nil {
+		slog.Error("error writing form field value", "key", key, "value", value)
+		return err
+	}
+
+	return nil
+}

test-harness/go.mod

@@ -1,4 +1,4 @@
-module github.com/TBD54566975/web5-spec
+module github.com/TBD54566975/sdk-development
 
 go 1.20
 

test-harness/reports/html.go

@@ -0,0 +1,62 @@
+package reports
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"golang.org/x/exp/slog"
+)
+
+func sanatizeHTML(dirty error) string {
+	clean := strings.ReplaceAll(dirty.Error(), "<", "&lt;")
+	clean = strings.ReplaceAll(clean, ">", "&gt;")
+	clean = strings.ReplaceAll(clean, "\n", "\\\\n")
+
+	return clean
+}
+
+type htmlTemplateInput struct {
+	Reports []Report
+	Tests   map[string][]string
+}
+
+func WriteHTML(reports []Report, filename string) error {
+	slog.Info("writing html report")
+
+	testmap := make(map[string]map[string]bool)
+	for _, report := range reports {
+		for category, tests := range report.Results {
+			if _, ok := tests[category]; !ok {
+				testmap[category] = map[string]bool{}
+			}
+
+			for test := range tests {
+				testmap[category][test] = true
+			}
+		}
+	}
+
+	templateInput := htmlTemplateInput{
+		Reports: reports,
+		Tests:   make(map[string][]string),
+	}
+
+	for category, tests := range testmap {
+		for test := range tests {
+			templateInput.Tests[category] = append(templateInput.Tests[category], test)
+		}
+	}
+
+	f, err := os.Create(filename)
+	if err != nil {
+		return fmt.Errorf("error opening %s: %v", filename, err)
+	}
+	defer f.Close()
+
+	if err := htmlTemplates.ExecuteTemplate(f, "report-template.html", templateInput); err != nil {
+		return err
+	}
+
+	return nil
+}

test-harness/reports/junit.go

@@ -0,0 +1,55 @@
+package reports
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"time"
+)
+
+func durationToJunit(t time.Duration) string {
+	return fmt.Sprintf("%f", float64(t)/float64(time.Second))
+}
+
+type junitTemplateInput struct {
+	TimeTotal        time.Duration
+	TimePerTestSuite map[string]time.Duration
+	Report           Report
+}
+
+func WriteJunitToFile(report Report, filename string) error {
+	f, err := os.Create(filename)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	if err := WriteJunit(report, f); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func WriteJunit(report Report, writer io.Writer) error {
+	templateInput := junitTemplateInput{
+		Report:           report,
+		TimePerTestSuite: map[string]time.Duration{},
+	}
+
+	for testsuite, results := range report.Results {
+		var testsuiteTime time.Duration
+		for _, result := range results {
+			testsuiteTime = testsuiteTime + result.Time
+			templateInput.TimeTotal = templateInput.TimeTotal + result.Time
+		}
+
+		templateInput.TimePerTestSuite[testsuite] = testsuiteTime
+	}
+
+	if err := textTemplates.ExecuteTemplate(writer, "report-template.junit.xml", templateInput); err != nil {
+		return err
+	}
+
+	return nil
+}

test-harness/reports/md.go

@@ -0,0 +1,18 @@
+package reports
+
+import "os"
+
+func WriteMarkdown(report Report, filename string) error {
+	f, err := os.Create(filename)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	err = textTemplates.ExecuteTemplate(f, "report-template.md", report)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

test-harness/reports/report-template.html

@@ -28,12 +28,12 @@ <h2>{{ $category }}</h2>
                 {{ range $.Reports }}
                 <td>
                     {{ if not (index (index .Results $category) $test) }}
-                    {{ index (index .Results $category) $test | getEmoji }}
+                    {{ (index (index .Results $category) $test).GetEmoji }}
                     {{ else }}
                     <details>
-                    <summary>{{ index (index .Results $category) $test | getEmoji }}</summary>
+                    <summary>{{ (index (index .Results $category) $test).GetEmoji }}</summary>
                     <ul>
-                        {{ range index (index .Results $category) $test }}<li>{{ . }}</li>{{ end }}
+                        {{ range (index (index .Results $category) $test).Errors }}<li>{{ . }}</li>{{ end }}
                     </ul>
                     {{ end }}
                 </td>{{ end }}

test-harness/reports/report-template.junit.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites time="{{ .TimeTotal | durationToJunit }}">
+{{ range $testSuiteName, $tests := .Report.Results }}
+    <testsuite name="{{ $testSuiteName }}" tests="{{ $tests | len }}" time="{{ index $.TimePerTestSuite $testSuiteName | durationToJunit }}">
+        {{ range $testCaseName, $result := $tests }}
+        <testcase name="{{ $testCaseName }}" classname="{{ $testSuiteName }}" time="{{ $result.Time | durationToJunit }}"{{ if $result.Errors }}>
+            {{ if $result.IsSkipped }}
+            <skipped message="test is not supported by this SDK" />
+            {{ else }}
+            <failure message="test did not pass" type="AssertionError">
+                {{ range $_, $err := $result.Errors }}{{ $err }}{{ end }}
+            </failure> 
+            {{ end }}
+        </testcase>
+        {{ else }}/>{{ end}}
+        {{ end }}
+    </testsuite>
+{{ end }}
+</testsuites>

test-harness/reports/report-template.md

@@ -8,6 +8,6 @@ SDK: [{{ .TestServerID.Name }}]({{ .TestServerID.Url }}) ({{ .TestServerID.Langu
 
 | Feature | Result |
 | ------- | ------ |{{ range $test, $result := $results }}
-| {{ $test }} | {{ $result | getEmoji}}{{ if $result }} <ul>{{ range $result }}<li><pre>{{ . | sanatizeHTML }}</pre></li>{{ end }}</ul>{{ end }} |{{ end }}
+| {{ $test }} | {{ $result.GetEmoji }}{{ if $result.Errors }} <ul>{{ range $result.Errors }}<li><pre>{{ . | sanatizeHTML }}</pre></li>{{ end }}</ul>{{ end }} |{{ end }}
 
 {{ end }}

test-harness/reports/report-template.txt

@@ -3,5 +3,5 @@ web5 spec conformance report for {{ .TestServerID.Name }} ({{ .TestServerID.Url
 {{ range $groupName, $results := .Results }}
 {{ $groupName }}
 ======================{{ range $test, $result := $results }}
-{{ $test }}: {{ $result | getEmoji }} {{ if $result }}fail: {{ $result }}{{ else }}pass{{ end }}{{ end }}
+{{ $test }}: {{ $result.GetEmoji }} {{ if $result.Errors }}fail: {{ $result.Errors }}{{ else }}pass{{ end }}{{ end }}
 {{ end }}