Skip to content

Commit

Permalink
fix status checks for multiple modules
Browse files Browse the repository at this point in the history
Issue:

If you have workloads in different namespaces managed in different modules,
Skaffold will only run status checks on the first module it encounters. This
can be a problem when you have dependencies between the modules, where workloads
in one must be healthy before moving onto the next (e.g. CRD operators).

For example,

```yaml
apiVersion: skaffold/v4beta8
kind: Config
metadata:
  name: a
manifests:
  kustomize:
    paths:
      - ./deploy-a.yaml
deploy:
  kubectl:
    flags:
      apply:
        -  --server-side
---
apiVersion: skaffold/v4beta8
kind: Config
metadata:
  name: b
manifests:
  kustomize:
    paths:
      - ./deploy-b.yaml
deploy:
  kubectl:
    flags:
      apply:
        -  --server-side
```

Where `deploy-a.yaml` looks something like:

```yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: a
  namespace: a
spec:
  template:
    spec:
      containers:
        - name: default
          image: ...
          command: ["sleep", "inf"]
          livenessProbe:
            exec:
              command: ["true"]
            initialDelaySeconds: 15
            periodSeconds: 5
```

And `deploy-b.yaml` looks something like:

```yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: b
  namespace: b
spec:
  template:
    spec:
      containers:
        - name: default
          image: ...
          command: ["sleep", "inf"]
          livenessProbe:
            exec:
              command: ["true"]
            initialDelaySeconds: 15
            periodSeconds: 5
```

Notes:
  - There's a Deployer created per config/module
  - A Deployer holds a reference to a list of namespaces to monitor that it
    updates as it deploys resources
  - A reference to this list is also passed to the Status Monitor that it uses
    to figure our what resources to monitor
  - But the Status Monitor is deduped per kubecontext, which leads to a mismatch
    between the reference to the namespace list for subsequent modules (assuming
    they use the same kubecontext)
  - There should be a Status Monitor per Deployer
  - Dedup on the config name (those are unique right?)

Also added a Flake to this project, making it easy to install/run Skaffold with Nix:

```sh
nix run github:<org>/skaffold/<rev-or-ref>
```
  • Loading branch information
jashandeep-sohi committed Mar 29, 2024
1 parent 6bff175 commit dbc2111
Show file tree
Hide file tree
Showing 4 changed files with 191 additions and 6 deletions.
119 changes: 119 additions & 0 deletions flake.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

65 changes: 65 additions & 0 deletions flake.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
{
description = "Easy and Repeatable Kubernetes Development";

inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";

flake-parts.url = "github:hercules-ci/flake-parts";

gomod2nix.url = "github:nix-community/gomod2nix";
gomod2nix.inputs.nixpkgs.follows = "nixpkgs";
};

outputs = inputs@{ flake-parts, ... }:
flake-parts.lib.mkFlake { inherit inputs; } {

systems = [ "x86_64-linux" "aarch64-linux" "aarch64-darwin" "x86_64-darwin" ];

perSystem = { pkgs, inputs', ... }: {

packages.default = let
buildDate = with inputs; "${self.lastModifiedDate or self.lastModified or "unknown"}";
version = with inputs; "${self.shortRev or self.dirtyShortRev or buildDate}";
in inputs'.gomod2nix.legacyPackages.buildGoApplication {
pname = "skaffold";

inherit version;

src = inputs.self;

modules = null;

subPackages = ["cmd/skaffold"];

ldflags = let
p = "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold";
in [
"-s" "-w"
"-X ${p}/version.version=v${version}"
"-X ${p}/version.gitCommit=${inputs.self.rev or inputs.self.dirtyRev or "unkown"}"
"-X ${p}/version.buildDate=${buildDate}"
];

nativeBuildInputs = with pkgs; [ installShellFiles makeWrapper ];

installCheckPhase = ''
$out/bin/skaffold version | grep ${version} > /dev/null
'';

postInstall = ''
wrapProgram $out/bin/skaffold --set SKAFFOLD_UPDATE_CHECK false
installShellCompletion --cmd skaffold \
--bash <($out/bin/skaffold completion bash) \
--zsh <($out/bin/skaffold completion zsh)
'';

meta = {
homepage = "https://github.com/GoogleContainerTools/skaffold";
};
};

packages.gomod2nix = inputs'.gomod2nix.packages.default;
};
};
}
10 changes: 5 additions & 5 deletions pkg/skaffold/deploy/component/kubernetes/component.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ func newLogger(config k8slogger.Config, cli *kubectl.CLI, podSelector kubernetes
return k8slogger.NewLogAggregator(cli, podSelector, namespaces, config)
}

func newMonitor(cfg k8sstatus.Config, kubeContext string, labeller *label.DefaultLabeller, namespaces *[]string, customResourceSelectors []manifest.GroupKindSelector) k8sstatus.Monitor {
func newMonitor(cfg k8sstatus.Config, dedupKey string, labeller *label.DefaultLabeller, namespaces *[]string, customResourceSelectors []manifest.GroupKindSelector) k8sstatus.Monitor {
if customResourceSelectors == nil {
customResourceSelectors = []manifest.GroupKindSelector{}
}
Expand All @@ -102,15 +102,15 @@ func newMonitor(cfg k8sstatus.Config, kubeContext string, labeller *label.Defaul
if k8sMonitor == nil {
k8sMonitor = make(map[string]k8sstatus.Monitor)
}
if k8sMonitor[kubeContext] == nil {
if k8sMonitor[dedupKey] == nil {
enabled := cfg.StatusCheck()
if enabled != nil && !*enabled { // assume disabled only if explicitly set to false
k8sMonitor[kubeContext] = &k8sstatus.NoopMonitor{}
k8sMonitor[dedupKey] = &k8sstatus.NoopMonitor{}
} else {
k8sMonitor[kubeContext] = k8sstatus.NewStatusMonitor(cfg, labeller, namespaces, customResourceSelectors)
k8sMonitor[dedupKey] = k8sstatus.NewStatusMonitor(cfg, labeller, namespaces, customResourceSelectors)
}
}
return k8sMonitor[kubeContext]
return k8sMonitor[dedupKey]
}

func newSyncer(cli *kubectl.CLI, namespaces *[]string, formatter k8slogger.Formatter) sync.Syncer {
Expand Down
3 changes: 2 additions & 1 deletion pkg/skaffold/deploy/kubectl/kubectl.go
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ func NewDeployer(cfg Config, labeller *label.DefaultLabeller, d *latest.KubectlD
podSelector := kubernetes.NewImageList()
kubectl := NewCLI(cfg, d.Flags, defaultNamespace)
namespaces, err := deployutil.GetAllPodNamespaces(cfg.GetNamespace(), cfg.GetPipelines())

if err != nil {
olog.Entry(context.TODO()).Warn("unable to parse namespaces - deploy might not work correctly!")
}
Expand Down Expand Up @@ -134,7 +135,7 @@ func NewDeployer(cfg Config, labeller *label.DefaultLabeller, d *latest.KubectlD
debugger: component.NewDebugger(cfg.Mode(), podSelector, &namespaces, cfg.GetKubeContext()),
imageLoader: component.NewImageLoader(cfg, kubectl.CLI),
logger: logger,
statusMonitor: component.NewMonitor(cfg, cfg.GetKubeContext(), labeller, &namespaces, customResourceSelectors),
statusMonitor: component.NewMonitor(cfg, configName, labeller, &namespaces, customResourceSelectors),
syncer: component.NewSyncer(kubectl.CLI, &namespaces, logger.GetFormatter()),
manifestsNamespaces: &manifestsNamespaces,
hookRunner: hooks.NewDeployRunner(kubectl.CLI, d.LifecycleHooks, &namespaces, logger.GetFormatter(), hooks.NewDeployEnvOpts(labeller.GetRunID(), kubectl.KubeContext, namespaces), &manifestsNamespaces),
Expand Down

0 comments on commit dbc2111

Please sign in to comment.