-
Notifications
You must be signed in to change notification settings - Fork 665
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Showing
4,595 changed files
with
1,496,111 additions
and
3,708 deletions.
The diff you're trying to view is too large. We only load the first 3000 changed files.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| [codespell] | ||
| skip = .git,*.pdf,*.svg,go.sum,go.mod,*requirements.txt,gen | ||
| # some strings with unicodes, constructs like [o]utput | ||
| ignore-regex = ".*\\0[0-9][0-9].*"|json:"[^"]*"|\b[a-z]*\[[a-z]\][a-z]*\b | ||
| # some ad-hoc variable names etc | ||
| ignore-words-list = te,nd,querys,ser |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| name: Bump git tags | ||
|
|
||
| on: | ||
| workflow_call: | ||
| outputs: | ||
| next_version: | ||
| description: "Tag name" | ||
| value: ${{ jobs.bump_version.outputs.next_version }} | ||
| secrets: | ||
| FLYTE_BOT_PAT: | ||
| required: true | ||
| jobs: | ||
| bump_version: | ||
| name: Bump Version | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| next_version: ${{ steps.get_semver.outputs.next_version }} | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| with: | ||
| fetch-depth: '0' | ||
| # Bump patch version | ||
| - uses: rickstaa/action-get-semver@v1 | ||
| id: get_semver | ||
| with: | ||
| bump_level: "patch" | ||
| - name: Print current and next versions | ||
| run: | | ||
| echo "Current version: ${{ steps.get_semver.outputs.current_version }}" | ||
| echo "Next version: ${{ steps.get_semver.outputs.next_version }}" | ||
| # Generate all tags for all components | ||
| - uses: actions/github-script@v6 | ||
| with: | ||
| github-token: ${{ secrets.FLYTE_BOT_PAT }} | ||
| script: | | ||
| github.rest.git.createRef({ | ||
| owner: context.repo.owner, | ||
| repo: context.repo.repo, | ||
| ref: `refs/tags/${{ steps.get_semver.outputs.next_version }}`, | ||
| sha: context.sha | ||
| }) | ||
| const components = [ | ||
| "datacatalog", | ||
| "flyteadmin", | ||
| "flytecopilot", | ||
| "flyteplugins", | ||
| "flytepropeller", | ||
| "flytestdlib", | ||
| ]; | ||
| for (const c of components) { | ||
| github.rest.git.createRef({ | ||
| owner: context.repo.owner, | ||
| repo: context.repo.repo, | ||
| ref: `refs/tags/${c}/${{ steps.get_semver.outputs.next_version }}`, | ||
| sha: context.sha | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,179 @@ | ||
| name: Components Checks | ||
|
|
||
| on: | ||
| pull_request: | ||
| paths: | ||
| - 'datacatalog/**' | ||
| - 'flyteadmin/**' | ||
| - 'flytecopilot/**' | ||
| - 'flyteidl/**' | ||
| - 'flyteplugins/**' | ||
| - 'flytepropeller/**' | ||
| - 'flytestdlib/**' | ||
| push: | ||
| branches: | ||
| - master | ||
| paths: | ||
| - 'datacatalog/**' | ||
| - 'flyteadmin/**' | ||
| - 'flytecopilot/**' | ||
| - 'flyteidl/**' | ||
| - 'flyteplugins/**' | ||
| - 'flytepropeller/**' | ||
| - 'flytestdlib/**' | ||
| env: | ||
| GO_VERSION: "1.19" | ||
| PRIORITIES: "P0" | ||
| jobs: | ||
| unpack-envvars: | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| go-version: ${{ steps.step.outputs.go-version }} | ||
| steps: | ||
| - id: step | ||
| run: | | ||
| echo "go-version=${{ env.GO_VERSION }}" >> $GITHUB_OUTPUT | ||
| lint: | ||
| name: Lint | ||
| needs: | ||
| - unpack-envvars | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| component: | ||
| - datacatalog | ||
| - flyteadmin | ||
| # TODO(monorepo): Enable lint flytecopilot | ||
| # - flytecopilot | ||
| - flyteidl | ||
| - flyteplugins | ||
| - flytepropeller | ||
| - flytestdlib | ||
| uses: ./.github/workflows/lint.yml | ||
| with: | ||
| component: ${{ matrix.component }} | ||
| go-version: ${{ needs.unpack-envvars.outputs.go-version }} | ||
| unit-tests: | ||
| name: Unit Tests | ||
| needs: | ||
| - unpack-envvars | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| component: | ||
| - datacatalog | ||
| - flyteadmin | ||
| - flytecopilot | ||
| # TODO(monorepo): Enable flyteidl unit tests | ||
| # - flyteidl | ||
| - flyteplugins | ||
| - flytepropeller | ||
| - flytestdlib | ||
| uses: ./.github/workflows/unit-tests.yml | ||
| with: | ||
| component: ${{ matrix.component }} | ||
| go-version: ${{ needs.unpack-envvars.outputs.go-version }} | ||
| secrets: | ||
| FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} | ||
| docker-build: | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| component: | ||
| - datacatalog | ||
| - flyteadmin | ||
| - flytecopilot | ||
| - flytepropeller | ||
| name: Docker Build Images | ||
| uses: ./.github/workflows/component_docker_build.yml | ||
| with: | ||
| component: ${{ matrix.component }} | ||
|
|
||
| # TODO(monorepo): these tests are broken. They never test an actual change. | ||
| # endtoend: | ||
| # name: End2End Test | ||
| # needs: [ docker-build ] | ||
| # uses: ./.github/workflows/end2end.yml | ||
| # with: | ||
| # # Reusing the output of the matrix is ok as they are essentially writing the same value (i.e. the directory artifacts are written + run id) | ||
| # cache_key: ${{ needs.docker-build.outputs.cache_key }} | ||
| # priorities: "P0" | ||
|
|
||
| integration: | ||
| name: Integration Test | ||
| needs: | ||
| - docker-build | ||
| - unpack-envvars | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| component: | ||
| - flyteadmin | ||
| uses: ./.github/workflows/integration.yml | ||
| with: | ||
| component: ${{ matrix.component }} | ||
| cache_key: ${{ needs.docker-build.outputs.cache_key }} | ||
| go-version: ${{ needs.unpack-envvars.outputs.go-version }} | ||
|
|
||
| generate: | ||
| name: Check Go Generate | ||
| needs: | ||
| - unpack-envvars | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| component: | ||
| - datacatalog | ||
| - flyteadmin | ||
| - flytecopilot | ||
| - flytepropeller | ||
| uses: ./.github/workflows/go_generate.yml | ||
| with: | ||
| component: ${{ matrix.component }} | ||
| go-version: ${{ needs.unpack-envvars.outputs.go-version }} | ||
|
|
||
| bump-tags: | ||
| name: Bump git tags | ||
| # TODO(monorepo): skip this if author is flyte-bot? | ||
| if: ${{ github.event_name != 'pull_request' }} | ||
| needs: | ||
| - integration | ||
| - lint | ||
| - unit-tests | ||
| - generate | ||
| uses: ./.github/workflows/bump-tags.yml | ||
| secrets: | ||
| FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} | ||
|
|
||
| # TODO(monorepo): we are not going to release any binary | ||
| # goreleaser: | ||
| # name: Goreleaser | ||
| # needs: [ bump_version ] # Only to ensure it can successfully build | ||
| # uses: flyteorg/flytetools/.github/workflows/goreleaser.yml@master | ||
| # with: | ||
| # go-version: "1.19" | ||
| # secrets: | ||
| # FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} | ||
|
|
||
| push_docker_image: | ||
| name: Build & Push Image | ||
| needs: [ bump-tags ] | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| component: | ||
| - datacatalog | ||
| - flyteadmin | ||
| - flytecopilot | ||
| - flytepropeller | ||
| - flytescheduler | ||
| uses: ./.github/workflows/publish.yml | ||
| with: | ||
| version: ${{ needs.bump-tags.outputs.next_version }} | ||
| component: ${{ matrix.component }} | ||
| dockerfile: Dockerfile.${{ matrix.component }} | ||
| push: true | ||
| secrets: | ||
| FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} | ||
| FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| --- | ||
| name: Codespell | ||
|
|
||
| on: | ||
| push: | ||
| branches: [master] | ||
| pull_request: | ||
| branches: [master] | ||
|
|
||
| permissions: | ||
| contents: read | ||
|
|
||
| jobs: | ||
| codespell: | ||
| name: Check for spelling errors | ||
| runs-on: ubuntu-latest | ||
|
|
||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v3 | ||
| - name: Codespell | ||
| uses: codespell-project/actions-codespell@v2 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| name: Build Docker Image | ||
|
|
||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| component: | ||
| required: true | ||
| type: string | ||
| outputs: | ||
| cache_key: | ||
| description: "Docker Cache key" | ||
| value: ${{ jobs.build_docker.outputs.cache_key }} | ||
| jobs: | ||
| build_docker: | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| cache_key: ${{ steps.cache_key.outputs.cache_key }} | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v3 | ||
| - id: load-docker-cache | ||
| name: Load Docker Cache | ||
| uses: actions/cache@v3 | ||
| with: | ||
| path: /tmp/tmp/docker-images-${{ inputs.component }} | ||
| key: /tmp/docker-images-${{ github.run_id }}-${{ inputs.component }} | ||
| restore-keys: | | ||
| /tmp/docker-images- | ||
| - name: Set cache key output | ||
| id: cache_key | ||
| run: | | ||
| echo ::set-output name=cache_key::/tmp/docker-images-${{ github.run_id }} | ||
| - name: Prime docker cache | ||
| run: (docker load -i /tmp/tmp/docker-images/snapshot-builder.tar || true) && (docker load -i /tmp/tmp/docker-images/snapshot.tar || true) | ||
| - name: Build dockerfile | ||
| env: | ||
| # We are unable to leverage docker buildx here without changing the | ||
| # caching mechanism significantly. See here for the caching options | ||
| # available for docker buildx: https://docs.docker.com/engine/reference/commandline/buildx_build/#cache-from | ||
| # For now at least enable DOCKER_BUILDKIT for faster builds. Eventually we | ||
| # should rewrite this pipeline to use docker buildx with cache type=gha. | ||
| DOCKER_BUILDKIT: "1" | ||
| run: | | ||
| docker build -t ${{ github.repository_owner }}/${{ inputs.component }}:builder --target builder --cache-from=${{ github.repository_owner }}/${{ inputs.component }}:builder --file Dockerfile.${{ inputs.component }} . | ||
| docker build -t ${{ github.repository_owner }}/${{ inputs.component }}:latest --cache-from=${{ github.repository_owner }}/${{ inputs.component }}:builder --file Dockerfile.${{ inputs.component }} . | ||
| - name: Tag and cache docker image | ||
| run: | | ||
| mkdir -p /tmp/tmp/docker-images-${{ inputs.component }} && docker save ${{ github.repository_owner }}/${{ inputs.component }}:builder -o /tmp/tmp/docker-images-${{ inputs.component }}/snapshot-builder-${{ inputs.component }}.tar && docker save ${{ github.repository_owner }}/${{ inputs.component }}:latest -o /tmp/tmp/docker-images-${{ inputs.component }}/snapshot-${{ inputs.component }}.tar |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.