Skip to content

Commit

Permalink
Extend StreamsBootstrap model
Browse files Browse the repository at this point in the history
  • Loading branch information
@raminqaf
raminqaf committed Oct 21, 2024
1 parent f734f40 commit 0a5ace8
Show file tree
Hide file tree
Showing 10 changed files with 787 additions and 3 deletions.
291 changes: 291 additions & 0 deletions docs/docs/schema/defaults.json
View file

Large diffs are not rendered by default.

250 changes: 250 additions & 0 deletions docs/docs/schema/pipeline.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,16 @@
"title": "HelmRepoConfig",
"type": "object"
},
"ImagePullPolicy": {
"description": "Represents the different Kubernetes image pull policies.",
"enum": [
"Always",
"IfNotPresent",
"Never"
],
"title": "ImagePullPolicy",
"type": "string"
},
"InputTopicTypes": {
"description": "Input topic types.\n\n- INPUT: input topic\n- PATTERN: extra-topic-pattern or input-topic-pattern",
"enum": [
Expand All @@ -218,6 +228,27 @@
"title": "InputTopicTypes",
"type": "string"
},
"JavaOptions": {
"description": "JVM configuration options.",
"properties": {
"maxRAMPercentage": {
"default": 75,
"description": "Sets the maximum amount of memory that the JVM may use for the Java heap before applying ergonomics heuristics as a percentage of the maximum amount determined as described in the -XX:MaxRAM option",
"title": "Maxrampercentage",
"type": "integer"
},
"others": {
"description": "List of Java VM options passed to the streams app.",
"items": {
"type": "string"
},
"title": "Others",
"type": "array"
}
},
"title": "JavaOptions",
"type": "object"
},
"KafkaConnectorConfig": {
"additionalProperties": true,
"additional_properties": {
Expand Down Expand Up @@ -791,13 +822,35 @@
"title": "Files",
"type": "object"
},
"image": {
"description": "Docker image of the Kafka producer app.",
"title": "Image",
"type": "string"
},
"imagePullPolicy": {
"allOf": [
{
"$ref": "#/$defs/ImagePullPolicy"
}
],
"default": "Always",
"description": "Docker image pull policy."
},
"imageTag": {
"default": "latest",
"description": "Docker image tag of the streams-bootstrap app.",
"pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$",
"title": "Imagetag",
"type": "string"
},
"javaOptions": {
"allOf": [
{
"$ref": "#/$defs/JavaOptions"
}
],
"description": ""
},
"kafka": {
"allOf": [
{
Expand Down Expand Up @@ -828,6 +881,24 @@
"title": "Ports",
"type": "array"
},
"resources": {
"allOf": [
{
"$ref": "#/$defs/Resources"
}
],
"default": {
"limits": {
"cpu": "300m",
"memory": "2G"
},
"requests": {
"cpu": "100m",
"memory": "500Mi"
}
},
"description": "See https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/"
},
"secretFilesRefs": {
"description": "Mount existing secrets as volumes",
"items": {
Expand Down Expand Up @@ -862,6 +933,7 @@
}
},
"required": [
"image",
"kafka"
],
"title": "ProducerAppValues",
Expand Down Expand Up @@ -967,6 +1039,79 @@
"title": "ProducerStreamsConfig",
"type": "object"
},
"PrometheusConfig": {
"description": "Prometheus JMX exporter configuration.",
"properties": {
"jmx": {
"allOf": [
{
"$ref": "#/$defs/PrometheusJMXConfig"
}
],
"description": "The prometheus JMX exporter configuration."
}
},
"title": "PrometheusConfig",
"type": "object"
},
"PrometheusJMXConfig": {
"description": "Prometheus JMX exporter configuration.",
"properties": {
"enabled": {
"default": true,
"description": "Whether to install Prometheus JMX Exporter as a sidecar container and expose JMX metrics to Prometheus.",
"title": "Enabled",
"type": "boolean"
},
"image": {
"default": "solsson/kafka-prometheus-jmx-exporter@sha256",
"description": "Docker Image for Prometheus JMX Exporter container.",
"title": "Image",
"type": "string"
},
"imagePullPolicy": {
"allOf": [
{
"$ref": "#/$defs/ImagePullPolicy"
}
],
"default": "IfNotPresent",
"description": "Docker Image Pull Policy for Prometheus JMX Exporter container."
},
"imageTag": {
"default": "6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143",
"description": "Docker Image Tag for Prometheus JMX Exporter container.",
"title": "Imagetag",
"type": "string"
},
"port": {
"default": 5556,
"description": "JMX Exporter Port which exposes metrics in Prometheus format for scraping.",
"title": "Port",
"type": "integer"
},
"resources": {
"allOf": [
{
"$ref": "#/$defs/Resources"
}
],
"default": {
"limits": {
"cpu": "300m",
"memory": "2G"
},
"requests": {
"cpu": "100m",
"memory": "500Mi"
}
},
"description": "JMX Exporter resources configuration."
}
},
"title": "PrometheusJMXConfig",
"type": "object"
},
"ProtocolSchema": {
"description": "Represents the different Kubernetes protocols.\n\nhttps://kubernetes.io/docs/reference/networking/service-protocols/",
"enum": [
Expand Down Expand Up @@ -1044,6 +1189,62 @@
"title": "RepoAuthFlags",
"type": "object"
},
"ResourceLimits": {
"properties": {
"cpu": {
"pattern": "^\\d+m$",
"title": "Cpu",
"type": "string"
},
"memory": {
"pattern": "^\\d+[KMGi]+$",
"title": "Memory",
"type": "string"
}
},
"required": [
"cpu",
"memory"
],
"title": "ResourceLimits",
"type": "object"
},
"ResourceRequests": {
"properties": {
"cpu": {
"pattern": "^\\d+m$",
"title": "Cpu",
"type": "string"
},
"memory": {
"pattern": "^\\d+[KMGi]+$",
"title": "Memory",
"type": "string"
}
},
"required": [
"cpu",
"memory"
],
"title": "ResourceRequests",
"type": "object"
},
"Resources": {
"properties": {
"limits": {
"$ref": "#/$defs/ResourceLimits"
},
"requests": {
"$ref": "#/$defs/ResourceRequests"
}
},
"required": [
"requests",
"limits"
],
"title": "Resources",
"type": "object"
},
"ServiceConfig": {
"description": "Base model for configuring a service for the Kafka Streams application.",
"properties": {
Expand Down Expand Up @@ -1391,13 +1592,35 @@
"title": "Files",
"type": "object"
},
"image": {
"description": "Docker image of the Kafka producer app.",
"title": "Image",
"type": "string"
},
"imagePullPolicy": {
"allOf": [
{
"$ref": "#/$defs/ImagePullPolicy"
}
],
"default": "Always",
"description": "Docker image pull policy."
},
"imageTag": {
"default": "latest",
"description": "Docker image tag of the streams-bootstrap app.",
"pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$",
"title": "Imagetag",
"type": "string"
},
"javaOptions": {
"allOf": [
{
"$ref": "#/$defs/JavaOptions"
}
],
"description": ""
},
"kafka": {
"allOf": [
{
Expand Down Expand Up @@ -1441,6 +1664,32 @@
"title": "Ports",
"type": "array"
},
"prometheus": {
"allOf": [
{
"$ref": "#/$defs/PrometheusConfig"
}
],
"description": ""
},
"resources": {
"allOf": [
{
"$ref": "#/$defs/Resources"
}
],
"default": {
"limits": {
"cpu": "300m",
"memory": "2G"
},
"requests": {
"cpu": "100m",
"memory": "500Mi"
}
},
"description": "See https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/"
},
"secretFilesRefs": {
"description": "Mount existing secrets as volumes",
"items": {
Expand Down Expand Up @@ -1481,6 +1730,7 @@
}
},
"required": [
"image",
"kafka"
],
"title": "StreamsAppValues",
Expand Down
25 changes: 25 additions & 0 deletions kpops/components/common/kubernetes_model.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import enum

from pydantic import BaseModel, Field


class ServiceType(enum.Enum):
"""Represents the different Kubernetes service types.
Expand All @@ -22,3 +24,26 @@ class ProtocolSchema(enum.Enum):
TCP = "TCP"
UDP = "UDP"
SCTP = "SCTP"


class ImagePullPolicy(enum.Enum):
"""Represents the different Kubernetes image pull policies."""

ALWAYS = "Always"
IF_NOT_PRESENT = "IfNotPresent"
NEVER = "Never"


class ResourceLimits(BaseModel):
cpu: str = Field(pattern=r"^\d+m$")
memory: str = Field(pattern=r"^\d+[KMGi]+$")


class ResourceRequests(BaseModel):
cpu: str = Field(pattern=r"^\d+m$")
memory: str = Field(pattern=r"^\d+[KMGi]+$")


class Resources(BaseModel):
requests: ResourceRequests
limits: ResourceLimits
Loading

0 comments on commit 0a5ace8

Please sign in to comment.