From 075eeac61b105ba94dffb6f551a106e0ed1dd7a1 Mon Sep 17 00:00:00 2001
From: Mark Sagi-Kazar <mark.sagikazar@gmail.com>
Date: Sat, 4 May 2024 22:29:26 +0200
Subject: [PATCH 1/2] ci: generate python SDK using Dagger
Signed-off-by: Mark Sagi-Kazar <mark.sagikazar@gmail.com>
---
.github/workflows/generate-client-python.yaml | 34 +++++++------------
api/client/python/Dockerfile | 16 ---------
api/client/python/Makefile | 30 ----------------
api/client/python/config.yaml | 2 +-
ci/generate.go | 32 +++++++++++++++++
ci/main.go | 2 ++
6 files changed, 48 insertions(+), 68 deletions(-)
delete mode 100644 api/client/python/Dockerfile
delete mode 100644 api/client/python/Makefile
create mode 100644 ci/generate.go
diff --git a/.github/workflows/generate-client-python.yaml b/.github/workflows/generate-client-python.yaml
index aa3a2c45a..56aae0eff 100644
--- a/.github/workflows/generate-client-python.yaml
+++ b/.github/workflows/generate-client-python.yaml
@@ -1,4 +1,4 @@
-name: Python Client Generation
+name: Generate / Python SDK
permissions:
contents: write
@@ -17,31 +17,23 @@ jobs:
generate:
runs-on: ubuntu-latest
steps:
- - uses: actions/create-github-app-token@v1
- id: app-token
+ - name: Checkout repository
+ uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
- app-id: ${{ vars.BOT_APP_ID }}
- private-key: ${{ secrets.BOT_APP_PRIVATE_KEY }}
- - uses: actions/checkout@v4
- with:
- token: ${{ steps.app-token.outputs.token }}
ref: ${{ github.head_ref }}
- # Make sure the value of GITHUB_TOKEN will not be persisted in repo's config
- persist-credentials: false
- - name: Login to GitHub Container Registry
- uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
- with:
- registry: ghcr.io
- username: ${{ github.actor }}
- password: ${{ github.token }}
+
- name: Generate Client
- run: |
- make generate
- working-directory: ./api/client/python
+ uses: dagger/dagger-for-github@eba69b4dddb54eddfdb51a88eb7fd86957137630 # v5.4.0
+ with:
+ verb: call
+ args: --source .:default generate python-sdk -o api/client/python
+ cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
+ version: "0.11.2"
+
- name: Open Pull Request
uses: peter-evans/create-pull-request@v6
with:
- branch: openapi/python-client
+ branch: openapi/python-sdk
branch-suffix: short-commit-hash
delete-branch: true
commit-message: "chore(api): generate python client"
@@ -50,4 +42,4 @@ jobs:
labels: |
area/api
release-note/misc
- token: ${{ steps.app-token.outputs.token }}
+ token: ${{ secrets.BOT_GITHUB_TOKEN }}
diff --git a/api/client/python/Dockerfile b/api/client/python/Dockerfile
deleted file mode 100644
index f0c455a1a..000000000
--- a/api/client/python/Dockerfile
+++ /dev/null
@@ -1,16 +0,0 @@
-# We build our image as the official autorest Dockerfile is outdated
-# and not compatible with the latest autorest.
-# More specifically, the latest autorest npm package depends on
-# other Azure packages that require a higher node version.
-# Official image: https://github.com/Azure/autorest/blob/63ffe68961e24ed8aa59a2ca4c16a8019c271e45/docker/base/ubuntu/Dockerfile
-
-# Autorest is incompatible with latest node version
-FROM node:20-alpine
-
-# Install autorest
-RUN npm install -g autorest
-
-# Install python/pip
-RUN apk add --no-cache python3 py3-pip
-
-ENTRYPOINT [ "autorest" ]
diff --git a/api/client/python/Makefile b/api/client/python/Makefile
deleted file mode 100644
index e246ed891..000000000
--- a/api/client/python/Makefile
+++ /dev/null
@@ -1,30 +0,0 @@
-# A Self-Documenting Makefile: http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
-
-.PHONY: autorest-python
-autorest-python: ## Build and publish autorest-python image
- $(call print-target)
- docker buildx build --platform linux/amd64,linux/arm64 -t ghcr.io/openmeterio/autorest-python:latest --push .
-
-.PHONY: generate
-generate: ## Generate code
- $(call print-target)
- cp ../../openapi.yaml .
- docker run \
- --rm \
- -v "$$(pwd):/workdir" \
- -w /workdir \
- -t ghcr.io/openmeterio/autorest-python:latest config.yaml
- rm openapi.yaml
-
-.PHONY: help
-.DEFAULT_GOAL := help
-help:
- @grep -h -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
-
-# Variable outputting/exporting rules
-var-%: ; @echo $($*)
-varexport-%: ; @echo $*=$($*)
-
-define print-target
- @printf "Executing target: \033[36m$@\033[0m\n"
-endef
diff --git a/api/client/python/config.yaml b/api/client/python/config.yaml
index 1aae0bcf5..0b27c3cfb 100644
--- a/api/client/python/config.yaml
+++ b/api/client/python/config.yaml
@@ -2,7 +2,7 @@ title: OpenMeterClient
namespace: openmeter
python: true
black: true
-input-file: ./openapi.yaml
+input-file: ../../openapi.yaml
output-folder: ./src
clear-output-folder: true
verbose: true
diff --git a/ci/generate.go b/ci/generate.go
new file mode 100644
index 000000000..cd0157fdd
--- /dev/null
+++ b/ci/generate.go
@@ -0,0 +1,32 @@
+package main
+
+// Generate various artifacts.
+func (m *Ci) Generate() *Generate {
+ return &Generate{
+ Source: m.Source,
+ }
+}
+
+type Generate struct {
+ // +private
+ Source *Directory
+}
+
+// Generate the Python SDK.
+func (m *Generate) PythonSdk() *Directory {
+ // We build our image as the official autorest Dockerfile is outdated
+ // and not compatible with the latest autorest.
+ // More specifically, the latest autorest npm package depends on
+ // other Azure packages that require a higher node version.
+ // Official image: https://github.com/Azure/autorest/blob/63ffe68961e24ed8aa59a2ca4c16a8019c271e45/docker/base/ubuntu/Dockerfile
+
+ // Autorest is incompatible with latest node version
+ return dag.Container().
+ From("node:20-alpine").
+ WithExec([]string{"npm", "install", "-g", "autorest"}).
+ WithExec([]string{"apk", "add", "python3", "py3-pip"}).
+ WithDirectory("/work", m.Source.Directory("api")).
+ WithWorkdir("/work/client/python").
+ WithExec([]string{"autorest", "config.yaml"}).
+ Directory("/work/client/python")
+}
diff --git a/ci/main.go b/ci/main.go
index f24f791f9..42b44ce8d 100644
--- a/ci/main.go
+++ b/ci/main.go
@@ -71,6 +71,8 @@ func (m *Ci) Ci(ctx context.Context) error {
wrapSyncable(m.Build().helmChart("benthos-collector", "0.0.0").File()),
wrapSyncables(m.releaseAssets("ci")),
+
+ wrapSyncable(m.Generate().PythonSdk()),
)
return p.wait()
From e9ea5f7dfb5fdd416f2097fe137edff1b68dc9de Mon Sep 17 00:00:00 2001
From: Mark Sagi-Kazar <mark.sagikazar@gmail.com>
Date: Sun, 5 May 2024 12:19:11 +0200
Subject: [PATCH 2/2] ci: move pypi publishing to dagger
Signed-off-by: Mark Sagi-Kazar <mark.sagikazar@gmail.com>
---
.github/workflows/release.yaml | 3 ++-
ci/release.go | 27 ++++++++++++++++++++++++++-
dagger.json | 4 ++++
3 files changed, 32 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 9e15dce3b..dd081bb24 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -41,8 +41,9 @@ jobs:
with:
verb: call
module: github.com/${{ github.repository }}@${{ github.ref }}
- args: --ref ${{ github.ref }} release --version ${{ github.ref_name }} --github-actor ${{ github.actor }} --github-token env:GITHUB_TOKEN
+ args: --ref ${{ github.ref }} release --version ${{ github.ref_name }} --github-actor ${{ github.actor }} --github-token env:GITHUB_TOKEN --pypi-token env:PYPI_TOKEN
cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
version: "0.11.2"
env:
GITHUB_TOKEN: ${{ github.token }}
+ PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
diff --git a/ci/release.go b/ci/release.go
index 7bbdfd7a8..9495dafc2 100644
--- a/ci/release.go
+++ b/ci/release.go
@@ -5,9 +5,10 @@ import (
"errors"
"fmt"
"strings"
+ "time"
)
-func (m *Ci) Release(ctx context.Context, version string, githubActor string, githubToken *Secret) error {
+func (m *Ci) Release(ctx context.Context, version string, githubActor string, githubToken *Secret, pypiToken *Secret) error {
p := newPipeline(ctx)
p.addJobs(
@@ -38,6 +39,10 @@ func (m *Ci) Release(ctx context.Context, version string, githubActor string, gi
return err
},
+
+ func(ctx context.Context) error {
+ return m.publishPythonSdk(ctx, version, pypiToken)
+ },
)
return p.wait()
@@ -94,3 +99,23 @@ func (m *Ci) binaryArchive(version string, platform Platform) *File {
WithFile("", m.Source.File("LICENSE")),
)
}
+
+func (m *Ci) publishPythonSdk(ctx context.Context, version string, pypiToken *Secret) error {
+ _, err := dag.Python(PythonOpts{
+ Container: dag.Python(PythonOpts{Container: dag.Container().From("pypy:3.10-slim")}).
+ WithPipCache(dag.CacheVolume("openmeter-pip")).
+ Container().
+ WithExec([]string{"pip", "--disable-pip-version-check", "install", "pipx"}).
+ WithExec([]string{"pipx", "install", "poetry"}),
+ }).
+ WithSource(m.Source.Directory("api/client/python")). // TODO: generate SDK on the fly?
+ Container().
+ WithExec([]string{"poetry", "install"}).
+ WithExec([]string{"poetry", "version", version}).
+ WithSecretVariable("POETRY_PYPI_TOKEN_PYPI", pypiToken).
+ WithEnvVariable("CACHE_BUSTER", time.Now().Format(time.RFC3339Nano)).
+ WithExec([]string{"poetry", "publish", "--build"}).
+ Sync(ctx)
+
+ return err
+}
diff --git a/dagger.json b/dagger.json
index c4add0dbf..8590f88df 100644
--- a/dagger.json
+++ b/dagger.json
@@ -40,6 +40,10 @@
"name": "kafka",
"source": "github.com/sagikazarmark/daggerverse/kafka@9a3074edb9cb21746ea2f9dcdd018d520b47f2e6"
},
+ {
+ "name": "python",
+ "source": "github.com/sagikazarmark/daggerverse/python@d8b5409dc45cc2242792566059f18cb1e5488629"
+ },
{
"name": "spectral",
"source": "github.com/sagikazarmark/daggerverse/spectral@9a3074edb9cb21746ea2f9dcdd018d520b47f2e6"