From e6d219423f49a2af9663957b858e892b957125de Mon Sep 17 00:00:00 2001 From: Pavlos Tzianos Date: Sun, 28 Jan 2024 22:37:10 +0000 Subject: [PATCH] Update docs to reflect new workflows Signed-off-by: Pavlos Tzianos --- docs/contributers/DEVELOPING.md | 4 +- docs/extending/example-producer/BUILD.sample | 47 ----- docs/extending/example-producer/main.go | 5 +- docs/extending/example-producer/main_test.go | 3 +- docs/getting-started/installation.md | 195 ++++++++++++++++++ docs/getting-started/kind.md | 132 ------------ docs/getting-started/please-k3d.md | 50 ----- .../tutorials/running-demos.md | 22 -- 8 files changed, 202 insertions(+), 256 deletions(-) delete mode 100644 docs/extending/example-producer/BUILD.sample create mode 100644 docs/getting-started/installation.md delete mode 100644 docs/getting-started/kind.md delete mode 100644 docs/getting-started/please-k3d.md delete mode 100644 docs/getting-started/tutorials/running-demos.md diff --git a/docs/contributers/DEVELOPING.md b/docs/contributers/DEVELOPING.md index 2dc21baf4..32c6cd7e3 100644 --- a/docs/contributers/DEVELOPING.md +++ b/docs/contributers/DEVELOPING.md @@ -20,13 +20,13 @@ We provide a complete development environment using the [Please build system](ht 4. Run formatters. ```bash - $ ./pleasew fmt-all + $ make fmt ``` 5. Run linters. ```bash - $ ./pleasew lint + $ make lint ``` #### Cleaning Up diff --git a/docs/extending/example-producer/BUILD.sample b/docs/extending/example-producer/BUILD.sample deleted file mode 100644 index 70e7be24c..000000000 --- a/docs/extending/example-producer/BUILD.sample +++ /dev/null @@ -1,47 +0,0 @@ -subinclude( - "//build/defs:buildkit", - "///k8s//build/defs:k8s", - "//build/defs:dracon", -) - -go_binary( - name = "example-tool-parser", - srcs = [ - "main.go", - ], - deps = [ - "//api/proto/v1", - "//components/producers", - ], -) - -go_test( - name = "example-tool-parser_test", - srcs = [ - "main.go", - "main_test.go", - ], - deps = [ - "//api/proto/v1", - "//components/producers", - "//third_party/go/github.com/stretchr/testify", - ], -) - -buildkit_distroless_image( - name = "image", - srcs = [":example-tool-parser"], - visibility = [ - "//examples/...", - ], -) - -dracon_component( - name = "example-tool", - images = [ - ":image", - "//third_party/docker/example/tool", - ], - task = "task.yaml", - visibility = ["//examples/pipelines/..."], -) diff --git a/docs/extending/example-producer/main.go b/docs/extending/example-producer/main.go index 519ab2523..e359eb7fe 100644 --- a/docs/extending/example-producer/main.go +++ b/docs/extending/example-producer/main.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "fmt" "log" @@ -20,7 +21,7 @@ func main() { } var results ExampleToolOut - if err := producers.ParseJSON(inFile, &results); err != nil { + if err := json.Unmarshal(inFile, &results); err != nil { log.Fatal(err) } @@ -52,7 +53,7 @@ func parseIssues(out *ExampleToolOut) []*v1.Issue { // ExampleToolOut represents the output of an ExampleTool run. type ExampleToolOut struct { - Issues []ExampleToolssue `json:"Issues"` + Issues []ExampleToolIssue `json:"Issues"` } // ExampleToolIssue represents an Example Tool Result. diff --git a/docs/extending/example-producer/main_test.go b/docs/extending/example-producer/main_test.go index 9666113b6..fa4c8f3df 100644 --- a/docs/extending/example-producer/main_test.go +++ b/docs/extending/example-producer/main_test.go @@ -7,6 +7,7 @@ import ( v1 "github.com/ocurity/dracon/api/proto/v1" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) const exampleOutput = ` @@ -34,7 +35,7 @@ const exampleOutput = ` func TestParseIssues(t *testing.T) { var results ExampleToolOut err := json.Unmarshal([]byte(exampleOutput), &results) - assert.Nil(t, err) + require.NoError(t, err) issues := parseIssues(&results) diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md new file mode 100644 index 000000000..3b1830e45 --- /dev/null +++ b/docs/getting-started/installation.md @@ -0,0 +1,195 @@ +# Getting Started with Dracon + +This guide will help to quickly setup Dracon on a Kubernetes cluster and get a pipeline running. +The first step is to create a dev Kubernetes cluster in order to deploy Tekton. We suggest you use +KiND to provision a local test cluster quickly. If you already have a K8s cluster then you can skip +directly to the + +## Setting up a [KinD](https://kind.sigs.k8s.io/) cluster + +KinD is is a tool for running local Kubernetes clusters using Docker container “nodes”. + +1. Create a KinD cluster named `dracon-demo` with its own Docker registry. You can use our Bash + script or you can check for more info the +[official documentation](https://kind.sigs.k8s.io/docs/user/quick-start/#creating-a-cluster): + +```bash +$ ./scripts/kind-with-registry.sh +``` + +> :warning: **Warning 1:** make sure that all pods are up an running before proceeding + +## Deploying Dracon dependencies + +1. When your Kubernetes cluster is ready to deploy the dependencies, make sure that you have set + the correct context and then run the following: + +```bash +$ make dev-deploy +``` + +The `dev-deploy` make target will deploy the following components: +a. ArangoDB +b. Nginx Ingress +c. Elastic Search Operator +d. Elastic Search +e. Kibana +f. MongoDB +g. Postgres +h. TektonCD and TektonCD Dashboard + +2. Expose the TektonCD Dashboard + +```bash +$ kubectl -n tekton-pipelines port-forward svc/tekton-dashboard 9097:9097 +``` + +3. Expose the Kibana Dashboard. + +```bash +# Use `kubectl port-forward ...` to access the Kibana UI: +$ kubectl -n dracon port-forward svc/dracon-kb-kibana-kb-http 5601:5601 +# You can obtain the password by examining the `dracon-es-elasticsearch-es-elastic-user` secret: +# The username is `elastic`. +$ kubectl -n dracon get secret dracon-es-elasticsearch-es-elastic-user \ + -o=jsonpath='{.data.elastic}' | \ + base64 -d && echo +``` + +4. Expose the ElasticSearch Dashboard + +```bash +# Use `kubectl port-forward ...` to access the Kibana UI: +$ kubectl -n dracon port-forward svc/dracon-kb-kibana-kb-http 5601:5601 +``` + +The username/password is the same as Kibana + +## Building all the components + +We use [Kustomize Components](https://github.com/kubernetes-sigs/kustomize/blob/master/examples/components.md) +to create composable Dracon Pipelines. These components are packaged into container images using +`make`. The following examples are using the local container registry used by the KiND cluster, but +make sure that you replace the URL with the registry URL that you are using, if you are using +something else: + +```bash +$ make publish-component-containers DOCKER_REPO=localhost:5001/ocurity/dracon +``` + +## Running your first Dracon pipeline + +1. Create the following simple Dracon Pipeline in your directory: + + ```yaml + --- + # ./kustomization.yaml + apiVersion: kustomize.config.k8s.io/v1beta1 + kind: Kustomization + + nameSuffix: -github-com-kubernetes-kubernetes + namespace: default + + resources: + - https://github.com/ocurity/dracon//components/base/ + + components: + - https://github.com/ocurity/dracon/components/sources/git/ + - https://github.com/ocurity/dracon/components/producers/aggregator/ + - https://github.com/ocurity/dracon/components/producers/golang-gosec/ + - https://github.com/ocurity/dracon/components/producers/golang-nancy/ + - https://github.com/ocurity/dracon/components/enrichers/aggregator/ + - https://github.com/ocurity/dracon/components/enrichers/deduplication/ + - https://github.com/ocurity/dracon/components/consumers/elasticsearch/ + ``` + +2. Run the following to create the Tekton Pipeline, Task, etc. resources on your cluster: + +```bash +$ kustomize build | kubectl apply -f - +# Note: you can just run the below to see the generated Tekton Pipeline resources +# $ kustomize build +``` + +3. Create the following Tekton PipelineRun file: + +```yaml +--- +# pipelinerun.yaml +# Run `kubectl create ...` with this file. +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + generateName: dracon-github-com-kubernetes-kubernetes- +spec: + pipelineRef: + name: dracon-github-com-kubernetes-kubernetes + params: + - name: repository_url + value: https://github.com/kubernetes/kubernetes.git + - name: consumer-elasticsearch-url + value: http://quickstart-es-http.default.svc:9200 + workspaces: + - name: source-code-ws + subPath: source-code + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +``` + +4. Create the PipelineRun resource: + +```bash +$ kubectl create -f pipelinerun.yaml +``` + +5. Observe the PipelineRun at http://localhost:8001/api/v1/namespaces/tekton-pipelines/services/tekton-dashboard:http/proxy/#/about + +6. Once the PipelineRun has finished, you can view the output in Kibana at http://localhost:5601. + + + + + +3. Deploy the Helm chart with the golang project + +```bash +$ helm upgrade golang-project-pipeline ./examples/pipelines/golang-project/ \ + --install \ + --namespace dracon +``` + +4. Start a pipeline run + +```bash +$ kubectl apply -n dracon -f ./examples/pipelines/golang-project/pipelinerun/pipelinerun.yaml +``` + +5. Expose the Tekton dashboard and Kibana UI + +```bash +$ kubectl -n tekton-pipelines port-forward svc/tekton-dashboard 9097:9097 +$ kubectl -n dracon port-forward svc/dracon-kb-kibana-kb-http 5601:5601 +``` + +6. Get the token to access Kibana + +```bash +# The username is `elastic`. +$ kubectl -n dracon get secret dracon-es-elasticsearch-es-elastic-user \ + -o=jsonpath='{.data.elastic}' \ + | base64 -d - \ + | xargs echo "$1" +``` + +7. Deploy the Golang pipeline + + + +8. Wait for the pipeline to finish running by monitoring it in http://localhost:9097. + +9. Once the pipelinerun has finished running you can view your results in Kibana: http://localhost:5601. diff --git a/docs/getting-started/kind.md b/docs/getting-started/kind.md deleted file mode 100644 index 57fe12097..000000000 --- a/docs/getting-started/kind.md +++ /dev/null @@ -1,132 +0,0 @@ -## Getting Started with Dracon on [KinD](https://kind.sigs.k8s.io/) - -KinD is is a tool for running local Kubernetes clusters using Docker container “nodes”. - -## Quick Guide - - -1. Create KinD cluster named `dracon-demo`. For more info, see [official documentation](https://kind.sigs.k8s.io/docs/user/quick-start/#creating-a-cluster): - - ```bash - $ kind create cluster --name dracon-demo - ``` - -2. Install Tekton Pipelines. For more info, see [official documentation](https://tekton.dev/docs/installation/pipelines/#installing-tekton-pipelines-on-kubernetes). - - ```bash - $ kubectl apply --filename https://storage.googleapis.com/tekton-releases/pipeline/latest/release.yaml - ``` - -3. **Optional**: Install Tekton Dashboard for a Web UI. For more info, see [official documentation](https://github.com/tektoncd/dashboard/blob/main/docs/install.md). - - ```bash - $ curl -sL https://raw.githubusercontent.com/tektoncd/dashboard/main/scripts/release-installer | \ - bash -s -- install latest --read-write - # Use `kubectl port-forward ...` so you can access Kubernetes services on your local machine. - $ kubectl -n tekton-pipelines port-forward svc/tekton-dashboard 9097:9097 - # Tekton Dashboard is now available at: http://localhost:9097 - ``` - -4. Create a Postgres DB to use as the Dracon deduplication DB: - - ```bash - # Create a StatefulSet and Service for the Dracon deduplication DB. In production, we recommend using a production-ready or managed Postgres deployment. - $ kubectl apply -f https://raw.githubusercontent.com/ocurity/dracon-community-pipelines/main/resources/deduplication-enricher-db.yaml - ``` - -5. Install ECK and create an Elasticsearch + Kibana Dashboards. For more info, see [official documentation](https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-deploy-eck.html). - - ```bash - # Create ECK CRDs. - $ kubectl create -f https://download.elastic.co/downloads/eck/2.6.1/crds.yaml - # Apply ECK operator resources. - $ kubectl apply -f https://download.elastic.co/downloads/eck/2.6.1/operator.yaml - # Create Elasticsearch. - $ kubectl apply -f https://raw.githubusercontent.com/ocurity/dracon-community-pipelines/main/resources/eck-elasticsearch.yaml - # Create Kibana. - $ kubectl apply -f https://raw.githubusercontent.com/ocurity/dracon-community-pipelines/main/resources/eck-kibana.yaml - # Use `kubectl port-forward ...` to access the Kibana UI: - $ kubectl port-forward svc/quickstart-kb-http 5601:5601 - # You can obtain the password by examining the `quickstart-es-elastic-user` secret: - # The username is `elastic`. - $ kubectl get secret quickstart-es-elastic-user \ - -o=jsonpath='{.data.elastic}' \ - | base64 -d - \ - | xargs echo "$1" - ``` - -### Composing a Pipeline - -We use [Kustomize Components](https://github.com/kubernetes-sigs/kustomize/blob/master/examples/components.md) to create composable Dracon Pipelines. - -1. Create the following simple Dracon Pipeline in your directory: - - ```yaml - --- - # ./kustomization.yaml - apiVersion: kustomize.config.k8s.io/v1beta1 - kind: Kustomization - - nameSuffix: -github-com-kubernetes-kubernetes - namespace: default - - resources: - - https://github.com/ocurity/dracon//components/base/ - - components: - - https://github.com/ocurity/dracon//components/sources/git/ - - https://github.com/ocurity/dracon//components/producers/aggregator/ - - https://github.com/ocurity/dracon//components/producers/golang-gosec/ - - https://github.com/ocurity/dracon//components/producers/golang-nancy/ - - https://github.com/ocurity/dracon//components/enrichers/aggregator/ - - https://github.com/ocurity/dracon//components/enrichers/deduplication/ - - https://github.com/ocurity/dracon//components/consumers/elasticsearch/ - ``` - -2. Run the following to create the Tekton Pipeline, Task, etc. resources on your cluster: - - ```bash - $ kustomize build | kubectl apply -f - - # Note: you can just run the below to see the generated Tekton Pipeline resources - # $ kustomize build - ``` - -3. Create the following Tekton PipelineRun file: - - ```yaml - --- - # pipelinerun.yaml - # Run `kubectl create ...` with this file. - apiVersion: tekton.dev/v1beta1 - kind: PipelineRun - metadata: - generateName: dracon-github-com-kubernetes-kubernetes- - spec: - pipelineRef: - name: dracon-github-com-kubernetes-kubernetes - params: - - name: repository_url - value: https://github.com/kubernetes/kubernetes.git - - name: consumer-elasticsearch-url - value: http://quickstart-es-http.default.svc:9200 - workspaces: - - name: source-code-ws - subPath: source-code - volumeClaimTemplate: - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - ``` - -4. Create the PipelineRun resource: - - ```bash - $ kubectl create -f pipelinerun.yaml - ``` - -5. Observe the PipelineRun at http://localhost:8001/api/v1/namespaces/tekton-pipelines/services/tekton-dashboard:http/proxy/#/about - -6. Once the PipelineRun has finished, you can view the output in Kibana at http://localhost:5601. diff --git a/docs/getting-started/please-k3d.md b/docs/getting-started/please-k3d.md deleted file mode 100644 index f557eaf98..000000000 --- a/docs/getting-started/please-k3d.md +++ /dev/null @@ -1,50 +0,0 @@ -## Getting Started with Dracon with Please on [k3d](https://k3d.io) - -k3d is a lightweight wrapper to run [k3s](https://k3s.io/) (Rancher Lab’s minimal Kubernetes distribution) in docker. - -k3d makes it very easy to create single- and multi-node k3s clusters in docker, e.g. for local development on Kubernetes. - -## Quick Guide - -1. Run the helper script which creates and configures a development k3d cluster. - - ```bash - $ ./pleasew dev - ``` - -2. Deploy supporting resources that Dracon uses. - - ```bash - $ ./pleasew dev_deploy - ``` - -3. Dracon is now ready to use. You can see available endpoints/ingresses by - running: - - ```bash - $ kubectl get ingress --all-namespaces -o jsonpath='{range .items[*]}{.spec.rules[0].host}{"\n"}{end}' \ - | sed 's/\(dracon\.localhost$\)/\1\:8080/g' \ - | sed 's/^/http:\/\//g' - ``` - Check out the [Running Demos Guide](/docs/getting-started/tutorials/running-demos.md) - -### Running an Example Pipeline - -1. Create the Pipeline in Tekton: - - ```bash - $ ./pleasew run //examples/pipelines/golang-project:golang-project_deploy - ``` - -2. Run the Pipeline in Tekton: - - ```bash - $ ./pleasew run //examples/pipelines/golang-project:golang-project_pipelinerun_deploy - ``` - -### Inspecting a Pipeline - -We can view the progress of a pipeline in 2 ways: - -- Visit and browse http://tekton.dracon.localhost:8080 -- Use `kubectl get pipelineruns` diff --git a/docs/getting-started/tutorials/running-demos.md b/docs/getting-started/tutorials/running-demos.md deleted file mode 100644 index 8c854035a..000000000 --- a/docs/getting-started/tutorials/running-demos.md +++ /dev/null @@ -1,22 +0,0 @@ -# Dracon Demos - -Example of running the the [Demo Dracon Pipelines](examples/pipelines/golang-project) where you get to see the results in Kibana at the end. - -## Prerequisites - -- You have followed the [Getting Started with k3d Guide](/docs/getting-started/k3d.md). - ---- - -## Tutorial - -1. You can run a demo pipeline with: - - ```bash - $ ./pleasew deploy //examples/pipelines/golang-project:pipeline - $ ./pleasew deploy //examples/pipelines/golang-project/pipelinerun:pipelinerun - ``` - -2. Wait for the pipeline to finish running by monitoring it in https://tekton.dracon.localhost:8443. - -3. Once the pipelinerun has finished running you can view your results in Kibana: https://kibana.dracon.localhost:8443.