Skip to content

Commit

Permalink
Kubernetes Operator for Nessie
Browse files Browse the repository at this point in the history
  • Loading branch information
adutra committed Jul 15, 2024
1 parent 849cfba commit 4f472c3
Show file tree
Hide file tree
Showing 193 changed files with 12,708 additions and 4 deletions.
73 changes: 69 additions & 4 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -88,14 +88,15 @@ jobs:
-x :nessie-quarkus:compileAll \
-x :nessie-server-admin-tool:compileAll \
-x :nessie-events-quarkus:compileAll \
-x :nessie-operator:compileAll \
--scan
- name: Gradle / Compile Quarkus
run: |
# 2 Retries - to mitigate https://github.com/gradle/gradle/issues/25751
./gradlew :nessie-quarkus:compileAll :nessie-server-admin-tool:compileAll :nessie-events-quarkus:compileAll --scan || \
./gradlew :nessie-quarkus:compileAll :nessie-server-admin-tool:compileAll :nessie-events-quarkus:compileAll --scan || \
./gradlew :nessie-quarkus:compileAll :nessie-server-admin-tool:compileAll :nessie-events-quarkus:compileAll --scan
./gradlew :nessie-quarkus:compileAll :nessie-server-admin-tool:compileAll :nessie-events-quarkus:compileAll :nessie-operator:compileAll --scan || \
./gradlew :nessie-quarkus:compileAll :nessie-server-admin-tool:compileAll :nessie-events-quarkus:compileAll :nessie-operator:compileAll --scan || \
./gradlew :nessie-quarkus:compileAll :nessie-server-admin-tool:compileAll :nessie-events-quarkus:compileAll :nessie-operator:compileAll --scan
- name: Gradle / Code checks
run: ./gradlew codeChecks --scan
Expand Down Expand Up @@ -141,7 +142,7 @@ jobs:
java-version: ${{ matrix.java-version }}

- name: Gradle / test
run: ./gradlew test :nessie-client:check -x :nessie-client:intTest -x :nessie-quarkus:test -x :nessie-server-admin-tool:test -x :nessie-events-quarkus:test --scan
run: ./gradlew test :nessie-client:check -x :nessie-client:intTest -x :nessie-quarkus:test -x :nessie-server-admin-tool:test -x :nessie-events-quarkus:test -x :nessie-operator:test --scan

- name: Capture Test Reports
uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4
Expand Down Expand Up @@ -196,6 +197,13 @@ jobs:
./gradlew :nessie-events-quarkus:test --scan || \
./gradlew :nessie-events-quarkus:test --scan
- name: Gradle / Test Quarkus Operator
run: |
# 2 Retries - to mitigate https://github.com/gradle/gradle/issues/25751
./gradlew :nessie-operator:test --scan || \
./gradlew :nessie-operator:test --scan || \
./gradlew :nessie-operator:test --scan
- name: Dump quarkus.log
if: ${{ failure() }}
run: |
Expand Down Expand Up @@ -263,6 +271,7 @@ jobs:
-x :nessie-quarkus:intTest \
-x :nessie-server-admin-tool:intTest \
-x :nessie-events-quarkus:intTest \
-x :nessie-operator:intTest \
$(cat ../persist-prjs.txt) \
$(cat ../storage-prjs.txt) \
$(cat ../spark-prjs.txt) \
Expand Down Expand Up @@ -544,6 +553,61 @@ jobs:
job-name: 'int-test-quarkus-events'
java-version: ${{ matrix.java-version }}

int-test-quarkus-operator:
name: CI intTest Quarkus Operator
runs-on: ubuntu-22.04
strategy:
max-parallel: 1
matrix:
java-version: ['17']
steps:
- uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
- name: Setup runner
uses: ./.github/actions/setup-runner
- name: Setup Java, Gradle
uses: ./.github/actions/dev-tool-java
with:
java-version: ${{ matrix.java-version }}

- name: Prepare Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-prepare
with:
java-version: ${{ matrix.java-version }}

- name: Gradle / intTest Operator
uses: gradle/actions/setup-gradle@v3
with:
arguments: |
:nessie-operator:intTest
--scan
- name: Dump quarkus.log
if: ${{ failure() }}
run: |
find . -path "**/build/quarkus.log" | while read ql ; do
echo "::group::Quarkus build log $ql"
cat $ql
echo "::endgroup::"
done
- name: Capture Test Reports
uses: actions/upload-artifact@v4
if: ${{ failure() }}
with:
name: ci-inttest-operator-reports
path: |
**/build/quarkus.log
**/build/reports/*
**/build/test-results/*
retention-days: 7

- name: Save partial Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-save
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
with:
job-name: 'int-test-quarkus-operator'
java-version: ${{ matrix.java-version }}

determine-jobs:
name: CI Determine jobs
runs-on: ubuntu-22.04
Expand Down Expand Up @@ -1067,6 +1131,7 @@ jobs:
- int-test-quarkus-server
- int-test-quarkus-tool
- int-test-quarkus-events
- int-test-quarkus-operator
strategy:
max-parallel: 2
matrix:
Expand Down
29 changes: 29 additions & 0 deletions NOTICE
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,35 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


---
dk.brics.automaton:automaton

Copyright (c) 2001-2022 Anders Moeller
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


---
io.github.crac:org-crac

Expand Down
1 change: 1 addition & 0 deletions bom/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ dependencies {
api(project(":nessie-minio-testcontainer"))
api(project(":nessie-nessie-testcontainer"))
api(project(":nessie-network-tools"))
api(project(":nessie-operator"))
api(project(":nessie-quarkus-auth"))
api(project(":nessie-quarkus-common"))
api(project(":nessie-server-admin-tool"))
Expand Down
2 changes: 2 additions & 0 deletions gradle/libs.versions.toml
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,7 @@ quarkus-bom = { module = "io.quarkus.platform:quarkus-bom", version.ref = "quark
quarkus-cassandra-bom = { module = "io.quarkus.platform:quarkus-cassandra-bom", version.ref = "quarkusPlatform" }
quarkus-google-cloud-services-bom = { module = "io.quarkus.platform:quarkus-google-cloud-services-bom", version.ref = "quarkusPlatform" }
quarkus-logging-sentry = { module = "io.quarkiverse.loggingsentry:quarkus-logging-sentry", version = "2.0.7" }
quarkus-operator-sdk-bom = { module = "io.quarkus.platform:quarkus-operator-sdk-bom", version.ref = "quarkusPlatform" }
rest-assured = { module = "io.rest-assured:rest-assured", version = "5.5.0" }
rocksdb-jni = { module = "org.rocksdb:rocksdbjni", version = "9.4.0" }
scala-library-v212 = { module = "org.scala-lang:scala-library", version = { strictly = "[2.12, 2.13[", prefer = "2.12.19" }}
Expand All @@ -148,6 +149,7 @@ spark-sql-v34-v212 = { module = "org.apache.spark:spark-sql_2_12", version = { s
spark-sql-v34-v213 = { module = "org.apache.spark:spark-sql_2_13", version = { strictly = "[3.4, 3.5[", prefer = "3.4.2"}}
spark-sql-v35-v212 = { module = "org.apache.spark:spark-sql_2_12", version = { strictly = "[3.5, 3.6[", prefer = "3.5.1"}}
spark-sql-v35-v213 = { module = "org.apache.spark:spark-sql_2_13", version = { strictly = "[3.5, 3.6[", prefer = "3.5.1"}}
sundr-builder-annotations = { module = "io.sundr:builder-annotations", version = "0.103.1" }
testcontainers-bom = { module = "org.testcontainers:testcontainers-bom", version = "1.19.8" }
testcontainers-keycloak = { module = "com.github.dasniko:testcontainers-keycloak", version = "3.4.0" }
threeten-extra = { module = "org.threeten:threeten-extra", version = "1.8.0" }
Expand Down
1 change: 1 addition & 0 deletions gradle/projects.main.properties
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ nessie-keycloak-testcontainer=testing/keycloak-container
nessie-nessie-testcontainer=testing/nessie-container
nessie-network-tools=tools/network
nessie-object-storage-mock=testing/object-storage-mock
nessie-operator=operator
nessie-quarkus-auth=servers/quarkus-auth
nessie-quarkus-common=servers/quarkus-common
nessie-quarkus=servers/quarkus-server
Expand Down
114 changes: 114 additions & 0 deletions operator/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@

VERSION ?= $(shell cat ../version.txt | sed -e 's/.*-SNAPSHOT/latest/g')
RELEASE_VERSION ?= $(shell cat ../version.txt | sed -e 's/-SNAPSHOT//g')

# CHANNELS define the bundle channels used in the bundle.
# Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable")
# To re-generate a bundle for other specific channels without changing the standard setup, you can:
# - use the CHANNELS as arg of the bundle target (e.g make bundle CHANNELS=candidate,fast,stable)
# - use environment variables to overwrite this value (e.g export CHANNELS="candidate,fast,stable")
ifneq ($(origin CHANNELS), undefined)
BUNDLE_CHANNELS := --channels=$(CHANNELS)
endif

# DEFAULT_CHANNEL defines the default channel used in the bundle.
# Add a new line here if you would like to change its default config. (E.g DEFAULT_CHANNEL = "stable")
# To re-generate a bundle for any other default channel without changing the default setup, you can:
# - use the DEFAULT_CHANNEL as arg of the bundle target (e.g make bundle DEFAULT_CHANNEL=stable)
# - use environment variables to overwrite this value (e.g export DEFAULT_CHANNEL="stable")
ifneq ($(origin DEFAULT_CHANNEL), undefined)
BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL)
endif
BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL)

# IMAGE_TAG_BASE defines the docker.io namespace and part of the image name for remote images.
# This variable is used to construct full image tags for bundle and catalog images.
IMAGE_TAG_BASE ?= ghcr.io/projectnessie/nessie-operator

# BUNDLE_IMG defines the image:tag used for the bundle.
# You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=<some-registry>/<project-name-bundle>:<tag>)
BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:$(VERSION)

# Image URL to use all building/pushing image targets
IMG ?= $(IMAGE_TAG_BASE):$(VERSION)

PULL_POLICY ?= $(shell [ "$(VERSION)" = "latest" ] && echo "Always" || echo "IfNotPresent")
PLATFORM ?= linux/$(shell arch)

all: docker-build

##@ General

# The help target prints out all targets with their descriptions organized
# beneath their categories. The categories are represented by '##@' and the
# target descriptions by '##'. The awk commands is responsible for reading the
# entire set of makefiles included in this invocation, looking for lines of the
# file as xyz: ## something, and then pretty-format the target and help. Then,
# if there's a line with ##@ something, that gets pretty-printed as a category.
# More info on the usage of ANSI control characters for terminal formatting:
# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
# More info on the awk command:
# http://linuxcommand.org/lc3_adv_awk.php

help: ## Display this help.
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

##@ Build

docker-build: ## Build docker image with the manager.
../gradlew :nessie-operator:spotlessApply :nessie-operator:clean :nessie-operator:build -x check \
-Dquarkus.container-image.build=true \
-Dquarkus.container-image.image=${IMG} \
-Dquarkus.jib.platforms=${PLATFORM} \
-Dquarkus.kubernetes.prometheus.generate-service-monitor=false \
-Dquarkus.kubernetes.image-pull-policy=${PULL_POLICY}

docker-push: ## Build and push docker image with the manager.
../gradlew :nessie-operator:spotlessApply :nessie-operator:clean :nessie-operator:build -x check \
-Dquarkus.container-image.build=true \
-Dquarkus.container-image.push=true \
-Dquarkus.container-image.image=${IMG} \
-Dquarkus.jib.platforms=${PLATFORM} \
-Dquarkus.kubernetes.prometheus.generate-service-monitor=false \
-Dquarkus.kubernetes.image-pull-policy=${PULL_POLICY}

##@ Deployment

install: ## Install CRDs into the K8s cluster specified in ~/.kube/config.
@$(foreach file, $(wildcard build/kubernetes/*-v1.yml), kubectl apply -f $(file);)

uninstall: ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config.
@$(foreach file, $(wildcard build/kubernetes/*-v1.yml), kubectl delete -f $(file);)

deploy: ## Deploy controller to the K8s cluster specified in ~/.kube/config.
kubectl apply -f build/kubernetes/kubernetes.yml

undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config.
kubectl delete -f build/kubernetes/kubernetes.yml

##@ Helm

helm-install: ## Install CRDs and the operator using Helm.
helm install nessie-operator build/helm -n nessie-operator

helm-upgrade: ## Upgrade CRDs and the operator using Helm.
helm upgrade nessie-operator build/helm -n nessie-operator

helm-uninstall: ## Uninstall CRDs and the operator using Helm.
helm uninstall nessie-operator -n nessie-operator

##@ Bundle

.PHONY: bundle
bundle: ## Generate bundle manifests and metadata, then validate generated files.
cat build/kubernetes/* | operator-sdk generate bundle -q --overwrite --version $(RELEASE_VERSION) $(BUNDLE_METADATA_OPTS)
operator-sdk bundle validate ./bundle
# TODO use quarkus

.PHONY: bundle-build
bundle-build: ## Build the bundle image.
docker build -f build/bundle/nessie-operator/bundle.Dockerfile -t $(BUNDLE_IMG) build/bundle/nessie-operator

.PHONY: bundle-push
bundle-push: ## Push the bundle image.
docker push $(BUNDLE_IMG)
17 changes: 17 additions & 0 deletions operator/PROJECT
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# Code generated by tool. DO NOT EDIT.
# This file is used to track the info used to scaffold your project
# and allow the plugins properly work.
# More info: https://book.kubebuilder.io/reference/project-config.html
domain: projectnessie.org
layout:
- quarkus.javaoperatorsdk.io/v1-alpha
projectName: nessie-operator
resources:
- api:
crdVersion: v1
namespaced: true
domain: projectnessie.org
group: nessie
kind: Nessie
version: v1alpha1
version: "3"
16 changes: 16 additions & 0 deletions operator/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# Kubernetes Operator for Nessie

This module is a Kubernetes Operator for Nessie.

**WARNING: This is a work in progress and is not ready for production use.**

The operator is designed to manage the lifecycle of Nessie instances in a Kubernetes cluster.

This project was bootstrapped using [Operator SDK]:

```bash
operator-sdk init --plugins=quarkus --domain=projectnessie.org --project-name=nessie-operator
operator-sdk create api --plugins=quarkus --group nessie --version=v1alpha1 --kind=Nessie
```

[Operator SDK]:https://sdk.operatorframework.io/docs/cli/operator-sdk/
Loading

0 comments on commit 4f472c3

Please sign in to comment.