Skip to content

Commit

Permalink
Update files
Browse files Browse the repository at this point in the history
  • Loading branch information
raminqaf committed Nov 25, 2024
1 parent 3770a8e commit c23cfb2
Show file tree
Hide file tree
Showing 11 changed files with 356 additions and 8 deletions.
13 changes: 13 additions & 0 deletions docs/docs/schema/defaults.json
Original file line number Diff line number Diff line change
Expand Up @@ -2309,6 +2309,19 @@
"title": "Readinessprobe",
"type": "object"
},
"replicaCount": {
"anyOf": [
{
"type": "integer"
},
{
"type": "null"
}
],
"default": null,
"description": "The number of Kafka Streams replicas.",
"title": "Replicacount"
},
"resources": {
"allOf": [
{
Expand Down
13 changes: 13 additions & 0 deletions docs/docs/schema/pipeline.json
Original file line number Diff line number Diff line change
Expand Up @@ -1945,6 +1945,19 @@
"title": "Readinessprobe",
"type": "object"
},
"replicaCount": {
"anyOf": [
{
"type": "integer"
},
{
"type": "null"
}
],
"default": null,
"description": "The number of Kafka Streams replicas.",
"title": "Replicacount"
},
"resources": {
"allOf": [
{
Expand Down
29 changes: 26 additions & 3 deletions docs/docs/user/references/cli-commands.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,10 @@ $ kpops [OPTIONS] COMMAND [ARGS]...
* `generate`: Generate enriched pipeline representation
* `init`: Initialize a new KPOps project.
* `manifest`: Render final resource representation
* `patch`: Render final resource representation
* `pause`: Pauses the pipeline
* `reset`: Reset pipeline steps
* `schema`: Generate JSON schema.
* `sync`: Render final resource representation

## `kpops clean`

Expand Down Expand Up @@ -170,14 +171,14 @@ $ kpops manifest [OPTIONS] PIPELINE_PATHS...
* `--verbose / --no-verbose`: Enable verbose printing [default: no-verbose]
* `--help`: Show this message and exit.

## `kpops patch`
## `kpops pause`

In addition to generate, render final resource representation for each pipeline step, e.g. Kubernetes manifests.

**Usage**:

```console
$ kpops patch [OPTIONS] PIPELINE_PATHS...
$ kpops pause [OPTIONS] PIPELINE_PATHS...
```

**Arguments**:
Expand Down Expand Up @@ -251,3 +252,25 @@ $ kpops schema [OPTIONS] SCOPE:{pipeline|defaults|config}
**Options**:

* `--help`: Show this message and exit.

## `kpops sync`

In addition to generate, render final resource representation for each pipeline step, e.g. Kubernetes manifests.

**Usage**:

```console
$ kpops sync [OPTIONS] PIPELINE_PATHS...
```

**Arguments**:

* `PIPELINE_PATHS...`: Paths to dir containing 'pipeline.yaml' or files named 'pipeline.yaml'. [env var: KPOPS_PIPELINE_PATHS;required]

**Options**:

* `--dotenv FILE`: Path to dotenv file. Multiple files can be provided. The files will be loaded in order, with each file overriding the previous one. [env var: KPOPS_DOTENV_PATH]
* `--config DIRECTORY`: Path to the dir containing config.yaml files [env var: KPOPS_CONFIG_PATH; default: .]
* `--environment TEXT`: The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development). [env var: KPOPS_ENVIRONMENT]
* `--verbose / --no-verbose`: Enable verbose printing [default: no-verbose]
* `--help`: Show this message and exit.
24 changes: 23 additions & 1 deletion kpops/api/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -386,7 +386,7 @@ def init(
init_project(path, config_include_opt)


def patch(
def sync(
pipeline_path: Path,
dotenv: list[Path] | None = None,
config: Path = Path(),
Expand All @@ -408,6 +408,28 @@ def patch(
return resources


def pause(
pipeline_path: Path,
dotenv: list[Path] | None = None,
config: Path = Path(),
environment: str | None = None,
verbose: bool = True,
) -> list[Resource]:
pipeline = generate(
pipeline_path=pipeline_path,
dotenv=dotenv,
config=config,
environment=environment,
verbose=verbose,
)
resources: list[Resource] = []

for component in pipeline.components:
resource = component.manifest_pause()
resources.append(resource)
return resources


def _create_pipeline(
pipeline_path: Path,
kpops_config: KpopsConfig,
Expand Down
28 changes: 26 additions & 2 deletions kpops/cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -342,15 +342,39 @@ def clean(
short_help="Render final resource representation",
help="In addition to generate, render final resource representation for each pipeline step, e.g. Kubernetes manifests.",
)
def patch(
def sync(
pipeline_paths: list[Path] = PIPELINE_PATHS_ARG,
dotenv: list[Path] | None = DOTENV_PATH_OPTION,
config: Path = CONFIG_PATH_OPTION,
environment: str | None = ENVIRONMENT,
verbose: bool = VERBOSE_OPTION,
):
for pipeline_file_path in collect_pipeline_paths(pipeline_paths):
resources = kpops.patch(
resources = kpops.sync(
pipeline_path=pipeline_file_path,
dotenv=dotenv,
config=config,
environment=environment,
verbose=verbose,
)
for resource in resources:
for rendered_manifest in resource:
print_yaml(rendered_manifest)


@app.command(
short_help="Pauses the pipeline",
help="In addition to generate, render final resource representation for each pipeline step, e.g. Kubernetes manifests.",
)
def pause(
pipeline_paths: list[Path] = PIPELINE_PATHS_ARG,
dotenv: list[Path] | None = DOTENV_PATH_OPTION,
config: Path = CONFIG_PATH_OPTION,
environment: str | None = ENVIRONMENT,
verbose: bool = VERBOSE_OPTION,
):
for pipeline_file_path in collect_pipeline_paths(pipeline_paths):
resources = kpops.pause(
pipeline_path=pipeline_file_path,
dotenv=dotenv,
config=config,
Expand Down
4 changes: 4 additions & 0 deletions kpops/components/base_components/pipeline_component.py
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,10 @@ def manifest_clean(self) -> Resource:
"""Render final component resources, e.g. Kubernetes manifests."""
return []

def manifest_pause(self) -> Resource:
"""Render final component resources, e.g. Kubernetes manifests."""
return []

async def deploy(self, dry_run: bool) -> None:
"""Deploy component, e.g. to Kubernetes cluster.
Expand Down
6 changes: 6 additions & 0 deletions kpops/components/streams_bootstrap/streams/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,7 @@ class StreamsAppValues(StreamsBootstrapValues):
The attributes correspond to keys and values that are used as values for the streams bootstrap helm chart.
:param replica_count: The number of Kafka Streams replicas.
:param kafka: streams-bootstrap kafka section
:param autoscaling: Kubernetes event-driven autoscaling config, defaults to None
:param stateful_set: Whether to use a StatefulSet instead of a Deployment to deploy the streams app.
Expand All @@ -313,6 +314,11 @@ class StreamsAppValues(StreamsBootstrapValues):
:param termination_grace_period_seconds: Delay for graceful application shutdown in seconds: https://pracucci.com/graceful-shutdown-of-kubernetes-pods.html
"""

replica_count: int | None = Field(
default=None,
description=describe_attr("replica_count", __doc__),
)

kafka: StreamsConfig = Field(
description=describe_attr("kafka", __doc__),
)
Expand Down
8 changes: 8 additions & 0 deletions kpops/components/streams_bootstrap/streams/streams_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -200,3 +200,11 @@ def manifest_clean(self) -> Resource:
self.namespace,
values,
)

def manifest_pause(self) -> Resource:
autoscaling = self.values.autoscaling
if autoscaling and autoscaling.enabled:
autoscaling.max_replicas = 0
else:
self.values.replica_count = 0
return self.manifest_deploy()
Loading

0 comments on commit c23cfb2

Please sign in to comment.