Incorporate tbdex vector results into report (#99)

Co-authored-by: Finn Herzfeld <[email protected]>

.gitignore

@@ -1 +1,2 @@
 _site
+.idea

bin/.go-1.21.5.pkg

@@ -0,0 +1 @@
+hermit

bin/README.hermit.md

@@ -0,0 +1,7 @@
+# Hermit environment
+
+This is a [Hermit](https://github.com/cashapp/hermit) bin directory.
+
+The symlinks in this directory are managed by Hermit and will automatically
+download and install Hermit itself as well as packages. These packages are
+local to this environment.

bin/activate-hermit

@@ -0,0 +1,21 @@
+#!/bin/bash
+# This file must be used with "source bin/activate-hermit" from bash or zsh.
+# You cannot run it directly
+#
+# THIS FILE IS GENERATED; DO NOT MODIFY
+
+if [ "${BASH_SOURCE-}" = "$0" ]; then
+  echo "You must source this script: \$ source $0" >&2
+  exit 33
+fi
+
+BIN_DIR="$(dirname "${BASH_SOURCE[0]:-${(%):-%x}}")"
+if "${BIN_DIR}/hermit" noop > /dev/null; then
+  eval "$("${BIN_DIR}/hermit" activate "${BIN_DIR}/..")"
+
+  if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ]; then
+      hash -r 2>/dev/null
+    fi
+
+    echo "Hermit environment $("${HERMIT_ENV}"/bin/hermit env HERMIT_ENV) activated"
+fi

bin/go

@@ -0,0 +1 @@
+.go-1.21.5.pkg

bin/gofmt

@@ -0,0 +1 @@
+.go-1.21.5.pkg

bin/hermit

@@ -0,0 +1,43 @@
+#!/bin/bash
+#
+# THIS FILE IS GENERATED; DO NOT MODIFY
+
+set -eo pipefail
+
+export HERMIT_USER_HOME=~
+
+if [ -z "${HERMIT_STATE_DIR}" ]; then
+  case "$(uname -s)" in
+  Darwin)
+    export HERMIT_STATE_DIR="${HERMIT_USER_HOME}/Library/Caches/hermit"
+    ;;
+  Linux)
+    export HERMIT_STATE_DIR="${XDG_CACHE_HOME:-${HERMIT_USER_HOME}/.cache}/hermit"
+    ;;
+  esac
+fi
+
+export HERMIT_DIST_URL="${HERMIT_DIST_URL:-https://github.com/cashapp/hermit/releases/download/stable}"
+HERMIT_CHANNEL="$(basename "${HERMIT_DIST_URL}")"
+export HERMIT_CHANNEL
+export HERMIT_EXE=${HERMIT_EXE:-${HERMIT_STATE_DIR}/pkg/hermit@${HERMIT_CHANNEL}/hermit}
+
+if [ ! -x "${HERMIT_EXE}" ]; then
+  echo "Bootstrapping ${HERMIT_EXE} from ${HERMIT_DIST_URL}" 1>&2
+  INSTALL_SCRIPT="$(mktemp)"
+  # This value must match that of the install script
+  INSTALL_SCRIPT_SHA256="180e997dd837f839a3072a5e2f558619b6d12555cd5452d3ab19d87720704e38"
+  if [ "${INSTALL_SCRIPT_SHA256}" = "BYPASS" ]; then
+    curl -fsSL "${HERMIT_DIST_URL}/install.sh" -o "${INSTALL_SCRIPT}"
+  else
+    # Install script is versioned by its sha256sum value
+    curl -fsSL "${HERMIT_DIST_URL}/install-${INSTALL_SCRIPT_SHA256}.sh" -o "${INSTALL_SCRIPT}"
+    # Verify install script's sha256sum
+    openssl dgst -sha256 "${INSTALL_SCRIPT}" | \
+      awk -v EXPECTED="$INSTALL_SCRIPT_SHA256" \
+      '$2!=EXPECTED {print "Install script sha256 " $2 " does not match " EXPECTED; exit 1}'
+  fi
+  /bin/bash "${INSTALL_SCRIPT}" 1>&2
+fi
+
+exec "${HERMIT_EXE}" --level=fatal exec "$0" -- "$@"

reports/README.md

@@ -2,8 +2,7 @@
 
 This tool reads junit XML reports from different SDKs and generates an HTML report showing which SDKs support which features.
 
-`./cmd/build-html` will iterate over the repos listed in `sdk.go` and download the most recent junit artifact. It will read
-all junit results from it and produce a report to `_site/index.html`.  For local testing, generate a
+`./cmd/build-html` will iterate over the repos listed in `sdk.go` and download the most recent junit artifact. It will read all junit results from it and produce a report to `_site/index.html`.  For local testing, generate a
 [GitHub Personal Access Token](https://github.com/settings/tokens?type=beta) and put it in an environment variable named
 `GITHUB_TOKEN`. It doesn't need any special permissions ("Public Repositories (read-only)"). Note that a GitHub app (explained below) can also be used.
 
@@ -14,3 +13,15 @@ For local testing, a [GitHub App](https://github.com/settings/apps) must be crea
 * `CICD_ROBOT_GITHUB_APP_PRIVATE_KEY` - this should be the contents of the private key, not the path to the file.
 * `CICD_ROBOT_GITHUB_APP_NAME` - this is used as a display name and should match the name in the URL of the edit page for the app.
 * `CICD_ROBOT_GITHUB_APP_INSTALLATION_ID` - click "Install App" on the sidebar while editing the app in GitHub to install it on your own account.
+
+## Tooling
+
+This project uses [hermit](https://cashapp.github.io/hermit/usage/get-started/), an open source toolchain manager, which pins and automatically downloads and installs tooling for a repo, including compiler toolchains, utilities, etc.
+
+To install hermit, run:
+
+```bash
+https://github.com/cashapp/hermit/releases/download/stable/install.sh | /bin/bash
+```
+
+If using goland or intellij, also install the hermit plugin via [these instructions](https://cashapp.github.io/hermit/usage/ide/).

reports/go.mod

@@ -1,6 +1,6 @@
 module github.com/TBD54566975/sdk-development/reports
 
-go 1.20
+go 1.21
 
 require (
 	github.com/bradleyfalzon/ghinstallation/v2 v2.8.0

reports/html.go

@@ -17,34 +17,51 @@ func sanatizeHTML(dirty error) string {
 }
 
 type htmlTemplateInput struct {
-	Reports []Report
-	Tests   map[string][]string
+	Reports    []Report
+	Web5Tests  map[string][]string
+	TbDEXTests map[string][]string
 }
 
 func WriteHTML(reports []Report, filename string) error {
 	slog.Info("writing html report", "reports", len(reports))
 
 	testmap := make(map[string]map[string]bool)
+	tbdexTestMap := make(map[string]map[string]bool)
 	for _, report := range reports {
 		for category, tests := range report.Results {
 			if _, ok := tests[category]; !ok {
-				testmap[category] = map[string]bool{}
+				if report.SDK.Type == "web5" {
+					testmap[category] = map[string]bool{}
+				} else {
+					tbdexTestMap[category] = map[string]bool{}
+				}
 			}
 
 			for test := range tests {
-				testmap[category][test] = true
+				if report.SDK.Type == "web5" {
+					testmap[category][test] = true
+				} else {
+					tbdexTestMap[category][test] = true
+				}
 			}
 		}
 	}
 
 	templateInput := htmlTemplateInput{
-		Reports: reports,
-		Tests:   make(map[string][]string),
+		Reports:    reports,
+		Web5Tests:  make(map[string][]string),
+		TbDEXTests: make(map[string][]string),
 	}
 
 	for category, tests := range testmap {
 		for test := range tests {
-			templateInput.Tests[category] = append(templateInput.Tests[category], test)
+			templateInput.Web5Tests[category] = append(templateInput.Web5Tests[category], test)
+		}
+	}
+
+	for category, tests := range tbdexTestMap {
+		for test := range tests {
+			templateInput.TbDEXTests[category] = append(templateInput.TbDEXTests[category], test)
 		}
 	}
 

reports/report-template.html

@@ -1,7 +1,7 @@
 <!DOCTYPE html>
 <html>
   <head>
-    <title>web5 spec compliance report</title>
+    <title>TBD SDKs spec compliance report</title>
     <meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <link rel="stylesheet" href="styles.css" />
@@ -11,7 +11,7 @@
     <main>
       <h1>web5 spec compliance report</h1>
 
-      {{ range $category, $tests := .Tests }}
+      {{ range $category, $tests := .Web5Tests }}
       <h2 id="{{ $category }}_table-caption">{{ $category }}</h2>
       <table aria-labelledby="{{ $category }}_table-caption">
         <colgroup>
@@ -53,6 +53,51 @@ <h2 id="{{ $category }}_table-caption">{{ $category }}</h2>
         {{ end }}
       </table>
       {{ end }}
+
+      <h1>tbdex spec compliance report</h1>
+
+      {{ range $category, $tests := .TbDEXTests }}
+      <h2 id="{{ $category }}_table-caption">{{ $category }}</h2>
+      <table aria-labelledby="{{ $category }}_table-caption">
+        <colgroup>
+          <col span="1" />
+          <col span="2" />
+        </colgroup>
+        <thead>
+        <tr>
+          <th scope="col">test vector</th>
+          {{ range $.Reports }}
+          <th scope="col">
+            <a target="_blank" href="https://github.com/{{ .SDK.Repo }}"
+            >{{ .SDK.Name }}</a
+            >
+          </th>
+          {{ end }}
+        </tr>
+        </thead>
+        <tbody>
+        {{ range $i, $test := $tests }}
+        <tr>
+          <th scope="row">{{ $test }}</th>
+          {{ range $_, $report := $.Reports }}
+          <td>
+            <details{{ if eq (len (index (index .Results $category) $test).Errors) 0 }} tabindex="-1"{{ end }}>
+            <summary{{ if eq (len (index (index .Results $category) $test).Errors) 0 }} role="paragraph"{{ end }}>
+            <span aria-label="{{ (index (index .Results $category) $test).GetEmojiAriaLabel }}">{{ (index (index .Results $category) $test).GetEmoji }}</span>
+            </summary>
+            <ul>
+              {{ range (index (index .Results $category) $test).Errors }}
+              <li>{{ . }}</li>
+              {{ end }}
+            </ul>
+            </details>
+          </td>
+          {{ end }}
+        </tr>
+        </tbody>
+        {{ end }}
+      </table>
+      {{ end }}
     </main>
   </body>
 </html>

reports/reports.go

@@ -37,6 +37,7 @@ type SDKMeta struct {
 	FeatureRegex *regexp.Regexp
 	VectorRegex  *regexp.Regexp
 	VectorPath   string
+	Type         string
 }
 
 type Report struct {

reports/sdks.go

@@ -20,6 +20,7 @@ var (
 			FeatureRegex: regexp.MustCompile(`Web5TestVectors(\w+)`),
 			VectorRegex:  regexp.MustCompile(`.* Web5TestVectors\w+ (\w+)`),
 			VectorPath:   "test-vectors",
+			Type:         "web5",
 		},
 		{
 			Name:         "web5-kt",
@@ -28,6 +29,25 @@ var (
 			FeatureRegex: regexp.MustCompile(`web5\.sdk\.\w+.Web5TestVectors(\w+)`),
 			VectorRegex:  regexp.MustCompile(`(\w+)\(\)`),
 			VectorPath:   "test-vectors",
+			Type:         "web5",
+		},
+		{
+			Name:         "tbdex-js",
+			Repo:         "TBD54566975/tbdex-js",
+			ArtifactName: "junit-results",
+			FeatureRegex: regexp.MustCompile(`TbdexTestVectors(\w+)`),
+			VectorRegex:  regexp.MustCompile(`.* TbdexTestVectors\w+ (\w+)`),
+			VectorPath:   "test-vectors",
+			Type:         "tbdex",
+		},
+		{
+			Name:         "tbdex-kt",
+			Repo:         "TBD54566975/tbdex-kt",
+			ArtifactName: "test-results",
+			FeatureRegex: regexp.MustCompile(`tbdex\.sdk\.\w+.TbdexTestVectors(\w+)`),
+			VectorRegex:  regexp.MustCompile(`(\w+)\(\)`),
+			VectorPath:   "test-vectors",
+			Type:         "tbdex",
 		},
 	}
 )

scripts/test-vector-validation/main.js

@@ -7,7 +7,7 @@ import { fileURLToPath } from 'node:url'
 const __filename = fileURLToPath(import.meta.url)
 const __dirname = path.dirname(__filename)
 
-const vectorsDir = `${__dirname}/../../web5-test-vectors`
+const vectorsDir = `${__dirname}/../../test-vectors`
 
 let vectorsSchema = fs.readFileSync(`${vectorsDir}/vectors.schema.json`, 'utf8')
 vectorsSchema = JSON.parse(vectorsSchema)