Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support one-click-demo mode #503

Merged
merged 3 commits into from
Nov 13, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
169 changes: 125 additions & 44 deletions cmd/api/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
// to ensure that exec-entrypoint and run can make use of them.

_ "k8s.io/client-go/plugin/pkg/client/auth"
"k8s.io/client-go/rest"

"github.com/spf13/cobra"
"k8s.io/apimachinery/pkg/runtime"
Expand Down Expand Up @@ -101,6 +102,15 @@ type managerOpts struct {
zapOpts *zap.Options
clusterDomain string

// when true, ngrok-op will allow required fields to be optional
// then it will go Ready and log errors about registration state due to missing required fields
// this is useful for marketplace installations where our users do not have a chance to add their required configuration
// yet we still want a 1-click install to work
//
// when false, ngrok-op will require all required fields to be present before going Ready
// and will log errors about missing required fields
oneClickDemoMode bool

// feature flags
enableFeatureIngress bool
enableFeatureGateway bool
Expand All @@ -126,7 +136,7 @@ func cmd() *cobra.Command {
c := &cobra.Command{
Use: "api-manager",
RunE: func(c *cobra.Command, args []string) error {
return runController(c.Context(), opts)
return startOperator(c.Context(), opts)
},
}

Expand All @@ -145,6 +155,7 @@ func cmd() *cobra.Command {
// TODO(operator-rename): Same as above, but for the manager name.
c.Flags().StringVar(&opts.managerName, "manager-name", "ngrok-ingress-controller-manager", "Manager name to identify unique ngrok ingress controller instances")
c.Flags().StringVar(&opts.clusterDomain, "cluster-domain", "svc.cluster.local", "Cluster domain used in the cluster")
c.Flags().BoolVar(&opts.oneClickDemoMode, "one-click-demo-mode", false, "Run the operator in one-click-demo mode (Ready, but not running)")

// feature flags
c.Flags().BoolVar(&opts.enableFeatureIngress, "enable-feature-ingress", true, "Enables the Ingress controller")
Expand All @@ -164,76 +175,91 @@ func cmd() *cobra.Command {
return c
}

func runController(ctx context.Context, opts managerOpts) error {
// startOperator starts the ngrok-op
func startOperator(ctx context.Context, opts managerOpts) error {
ctrl.SetLogger(zap.New(zap.UseFlagOptions(opts.zapOpts)))

buildInfo := version.Get()
setupLog.Info("starting api-manager", "version", buildInfo.Version, "commit", buildInfo.GitCommit)

// create default kubernetes config and clientset
k8sConfig := ctrl.GetConfigOrDie()
k8sClient, err := client.New(k8sConfig, client.Options{Scheme: scheme})
if err != nil {
return fmt.Errorf("unable to create k8s client: %w", err)
}

var ok bool
opts.namespace, ok = os.LookupEnv("POD_NAMESPACE")
if !ok {
return errors.New("POD_NAMESPACE environment variable should be set, but was not")
}

opts.ngrokAPIKey, ok = os.LookupEnv("NGROK_API_KEY")
if !ok {
return errors.New("NGROK_API_KEY environment variable should be set, but was not")
mgr, err := loadManager(ctx, k8sConfig, opts)
if err != nil {
return fmt.Errorf("unable to load manager: %w", err)
}

buildInfo := version.Get()
setupLog.Info("starting api-manager", "version", buildInfo.Version, "commit", buildInfo.GitCommit)

clientConfigOpts := []ngrok.ClientConfigOption{
ngrok.WithUserAgent(version.GetUserAgent()),
if opts.oneClickDemoMode {
return runOneClickDemoMode(ctx, opts, k8sClient, mgr)
}

ngrokClientConfig := ngrok.NewClientConfig(opts.ngrokAPIKey, clientConfigOpts...)
if opts.apiURL != "" {
u, err := url.Parse(opts.apiURL)
if err != nil {
setupLog.Error(err, "api-url must be a valid ngrok API URL")
}
ngrokClientConfig.BaseURL = u
}
setupLog.Info("configured API client", "base_url", ngrokClientConfig.BaseURL)
return runNormalMode(ctx, opts, k8sClient, mgr)
}

ngrokClientset := ngrokapi.NewClientSet(ngrokClientConfig)
options := ctrl.Options{
Scheme: scheme,
Metrics: server.Options{
BindAddress: opts.metricsAddr,
},
WebhookServer: webhook.NewServer(webhook.Options{Port: 9443}),
HealthProbeBindAddress: opts.probeAddr,
LeaderElection: opts.electionID != "",
LeaderElectionID: opts.electionID,
// runOneClickDemoMode runs the operator in a one-click demo mode, meaning:
// - the operator will start even if required fields are missing
// - the operator will log errors about missing required fields
// - the operator will go Ready and log errors about registration state due to missing required fields
func runOneClickDemoMode(ctx context.Context, opts managerOpts, k8sClient client.Client, mgr ctrl.Manager) error {
// register healthchecks
if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
return fmt.Errorf("error setting up readyz check: %w", err)
}
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
return fmt.Errorf("error setting up health check: %w", err)
}

if opts.ingressWatchNamespace != "" {
options.Cache = cache.Options{
DefaultNamespaces: map[string]cache.Config{
opts.ingressWatchNamespace: {},
},
// start a ticker to print demo log messages
go func() {
ticker := time.NewTicker(10 * time.Second)
for {
select {
case <-ctx.Done():
break
case <-ticker.C:
setupLog.Error(errors.New("Running in one-click-demo mode"), "Ready even if required fields are missing!")
setupLog.Info("The ngrok-operator is running in one-click-demo mode which means the operator is not actually reconciling resources.")
setupLog.Info("Please provide ngrok API key and ngrok Authtoken in your Helm values to run the operator for real.")
setupLog.Info("Please set `oneClickDemoMode: false` in your Helm values to run the operator for real.")
}
}
}()

setupLog.Info("starting api-manager in one-click-demo mode")
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
return fmt.Errorf("error starting api-manager: %w", err)
}

// create default config and clientset for use outside the mgr.Start() blocking loop
k8sConfig := ctrl.GetConfigOrDie()
k8sClient, err := client.New(k8sConfig, client.Options{Scheme: scheme})
return nil
}

// runNormalMode runs the operator in normal operation mode
func runNormalMode(ctx context.Context, opts managerOpts, k8sClient client.Client, mgr ctrl.Manager) error {
ngrokClientset, err := loadNgrokClientset(ctx, opts)
if err != nil {
return fmt.Errorf("unable to create k8s client: %w", err)
return fmt.Errorf("Unable to load ngrokClientSet: %w", err)
}


// TODO(hkatz) for now we are hiding the k8sop API regstration behind the bindings feature flag
if opts.enableFeatureBindings {
// register the k8sop in the ngrok API
if err := createKubernetesOperator(ctx, k8sClient, opts); err != nil {
return fmt.Errorf("unable to create KubernetesOperator: %w", err)
}
}

mgr, err := ctrl.NewManager(k8sConfig, options)
if err != nil {
return fmt.Errorf("unable to start api-manager: %w", err)
}

// k8sResourceDriver is the driver that will be used to interact with the k8s resources for all controllers
// but primarily for kinds Ingress, Gateway, and ngrok CRDs
var k8sResourceDriver *store.Driver
Expand Down Expand Up @@ -305,14 +331,69 @@ func runController(ctx context.Context, opts managerOpts) error {
return fmt.Errorf("error setting up health check: %w", err)
}

setupLog.Info("starting api-manager")
setupLog.Info("starting api-manager in normal mode")
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
return fmt.Errorf("error starting api-manager: %w", err)
}

return nil
}

// loadManager loads the controller-runtime manager with the provided options
func loadManager(ctx context.Context, k8sConfig *rest.Config, opts managerOpts) (manager.Manager, error) {
options := ctrl.Options{
Scheme: scheme,
Metrics: server.Options{
BindAddress: opts.metricsAddr,
},
WebhookServer: webhook.NewServer(webhook.Options{Port: 9443}),
HealthProbeBindAddress: opts.probeAddr,
LeaderElection: opts.electionID != "",
LeaderElectionID: opts.electionID,
}

if opts.ingressWatchNamespace != "" {
options.Cache = cache.Options{
DefaultNamespaces: map[string]cache.Config{
opts.ingressWatchNamespace: {},
},
}
}

mgr, err := ctrl.NewManager(k8sConfig, options)
if err != nil {
return nil, fmt.Errorf("unable to start api-manager: %w", err)
}

return mgr, nil
}

// loadNgrokClientset loads the ngrok API clientset from the environment and managerOpts
func loadNgrokClientset(ctx context.Context, opts managerOpts) (ngrokapi.Clientset, error) {
var ok bool
opts.ngrokAPIKey, ok = os.LookupEnv("NGROK_API_KEY")
if !ok {
return nil, errors.New("NGROK_API_KEY environment variable should be set, but was not")
}

clientConfigOpts := []ngrok.ClientConfigOption{
ngrok.WithUserAgent(version.GetUserAgent()),
}

ngrokClientConfig := ngrok.NewClientConfig(opts.ngrokAPIKey, clientConfigOpts...)
if opts.apiURL != "" {
u, err := url.Parse(opts.apiURL)
if err != nil {
setupLog.Error(err, "api-url must be a valid ngrok API URL")
}
ngrokClientConfig.BaseURL = u
}
setupLog.Info("configured API client", "base_url", ngrokClientConfig.BaseURL)

ngrokClientset := ngrokapi.NewClientSet(ngrokClientConfig)
return ngrokClientset, nil
}

// getK8sResourceDriver returns a new Driver instance that is seeded with the current state of the cluster.
func getK8sResourceDriver(ctx context.Context, mgr manager.Manager, options managerOpts) (*store.Driver, error) {
logger := mgr.GetLogger().WithName("cache-store-driver")
Expand Down
15 changes: 8 additions & 7 deletions helm/ngrok-operator/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,14 @@ To uninstall the chart:

### Common parameters

| Name | Description | Value |
| ------------------- | ------------------------------------------------------------------ | ----------------------------------------- |
| `nameOverride` | String to partially override generated resource names | `""` |
| `fullnameOverride` | String to fully override generated resource names | `""` |
| `description` | ngrok-operator description that will appear in the ngrok dashboard | `The official ngrok Kubernetes Operator.` |
| `commonLabels` | Labels to add to all deployed objects | `{}` |
| `commonAnnotations` | Annotations to add to all deployed objects | `{}` |
| Name | Description | Value |
| ------------------- | ------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------- |
| `nameOverride` | String to partially override generated resource names | `""` |
| `fullnameOverride` | String to fully override generated resource names | `""` |
| `description` | ngrok-operator description that will appear in the ngrok dashboard | `The official ngrok Kubernetes Operator.` |
| `commonLabels` | Labels to add to all deployed objects | `{}` |
| `commonAnnotations` | Annotations to add to all deployed objects | `{}` |
| `oneClickDemoMode` | If true, then the operator will startup without required fields or API registration, become Ready, but not actually be running | `false` |

### Image configuration

Expand Down
2 changes: 2 additions & 0 deletions helm/ngrok-operator/templates/agent/deployment.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
{{- if and .Values.ingress.enabled (not .Values.oneClickDemoMode) }}
{{- $component := "agent" }}
{{- $rbacChecksum := include (print $.Template.BasePath "/agent/rbac.yaml") . | sha256sum }}
{{- $agent := .Values.agent }}
Expand Down Expand Up @@ -124,3 +125,4 @@ spec:
volumes:
{{ toYaml .Values.extraVolumes | nindent 6 }}
{{- end }}
{{- end }}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{{- if .Values.bindings.enabled }}
{{- if and .Values.bindings.enabled (not .Values.oneClickDemoMode) }}
{{- $component := "bindings-forwarder" }}
{{- $rbacChecksum := include (print $.Template.BasePath "/bindings-forwarder/rbac.yaml") . | sha256sum }}
{{- $forwarder := .Values.bindings.forwarder }}
Expand Down
3 changes: 3 additions & 0 deletions helm/ngrok-operator/templates/controller-deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,9 @@ spec:
args:
- --release-name={{ .Release.Name }}
{{- include "ngrok-operator.manager.cliFeatureFlags" . | nindent 8 }}
{{- if .Values.oneClickDemoMode }}
- --one-click-demo-mode
{{- end }}
{{- if .Values.bindings.enabled }}
- --bindings-name={{ .Values.bindings.name }}
- --bindings-allowed-urls={{ join "," .Values.bindings.allowedURLs }}
Expand Down
3 changes: 3 additions & 0 deletions helm/ngrok-operator/tests/agent/deployment_test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ suite: test agent deployment
templates:
- agent/deployment.yaml
- agent/rbac.yaml
set:
ingress:
enabled: true
tests:
- it: Should match snapshot
asserts:
Expand Down
9 changes: 9 additions & 0 deletions helm/ngrok-operator/tests/controller-deployment_test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,15 @@ tests:
- contains:
path: spec.template.spec.containers[0].args
content: --bindings-allowed-urls=test.example,http://*
- it: Should pass one-click-demo mode if set
set:
oneClickDemoMode: true
template: controller-deployment.yaml
documentIndex: 0 # Document 0 is the deployment since its the first template
asserts:
- contains:
path: spec.template.spec.containers[0].args
content: --one-click-demo-mode
- it: Should pass log format argument if set
set:
log:
Expand Down
5 changes: 5 additions & 0 deletions helm/ngrok-operator/values.schema.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions helm/ngrok-operator/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,13 @@
## @param description ngrok-operator description that will appear in the ngrok dashboard
## @param commonLabels Labels to add to all deployed objects
## @param commonAnnotations Annotations to add to all deployed objects
## @param oneClickDemoMode If true, then the operator will startup without required fields or API registration, become Ready, but not actually be running
nameOverride: ""
fullnameOverride: ""
description: "The official ngrok Kubernetes Operator."
commonLabels: {}
commonAnnotations: {}
oneClickDemoMode: false
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

doesn't this need to be set to true in order for it to work without having to set any required fields? Or do these on click app marketplaces allow specifying a helm value like this by default and its just we can't require user specific values to be set like API crednetials?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Or do these on click app marketplaces allow specifying a helm value like this by default and its just we can't require user specific values to be set like API crednetials?

The latter: My understanding is we can provide a values.yaml but not a pop-up for required fields. So we'll set onClickDemoMode: true in those marketplace values.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ahh ok, thats good to know, thanks

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if we can provide default values i'm kinda surprised we didn't just default these installations' replica count to 0 and instruct you to scale up after provider credentials vs a no-op mode, but I don't think I have all the historic context here


##
## @section Image configuration
Expand Down
Loading