Merge pull request #6 from infracloudio/f/codebundle-rds-mysql-conn

F/codebundle rds mysql conn

.env

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+HELM_RELEASE_NAME=runwhen-local
+NAMESPACE=monitoring
+WORKSPACE_OWNER_EMAIL=[email protected]

.github/workflows/build.yaml

@@ -21,7 +21,7 @@ jobs:
   build-and-push-ghcr:
     runs-on: ubuntu-latest
     env:
-      codebundle: hello_world
+      codebundle: rds-mysql-conn-count
     steps:
       - name: Checkout
         uses: actions/checkout@v2

.gitignore

@@ -2,4 +2,5 @@
 __pycache__
 output.xml
 log.html
-report.html
+report.html
+test/sre-stack/*

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "test/sre-stack"]
+	path = test/sre-stack
+	url = https://github.com/infracloudio/sre-stack.git
+	branch = main

Dockerfile

@@ -9,7 +9,7 @@ RUN pip install -r /app/codecollection/requirements.txt
 
 # Install packages
 RUN apt-get update && \
-    apt install -y git && \
+    apt install -y git default-mysql-client && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* && \
     rm -rf /var/cache/apt
@@ -19,4 +19,6 @@ RUN chown 1000:0 -R $WORKDIR
 RUN chown 1000:0 -R /app/codecollection
 
 # Set the user to $USER
-USER python
+USER python
+ENV USER=python
+RUN chmod -R u+x /app/codecollection

Makefile

@@ -0,0 +1,25 @@
+GIT_TLD=$(shell git rev-parse --show-toplevel)
+include $(GIT_TLD)/test/sre-stack/.env
+include $(GIT_TLD)/.env
+include $(GIT_TLD)/test/sre-stack/makefile
+SRE_STACK_DIR := $(GIT_TLD)/test/sre-stack
+RUNWHEN_SETUP_SCRIPT_PATH=setup/runwhen-local/setup.sh
+
+RUNWHEN_REQUIRED_VARS := RUNWHEN_PLATFORM_TOKEN
+$(foreach var,$(RUNWHEN_REQUIRED_VARS),$(if $(value $(var)),,$(error $(var) is not set)))
+
+setup-sre-stack:
+	$(MAKE) setup -C $(SRE_STACK_DIR)
+
+cleanup-sre-stack:
+	$(MAKE) cleanup -C $(SRE_STACK_DIR)
+
+setup-runwhen: 
+	$(GIT_TLD)/$(RUNWHEN_SETUP_SCRIPT_PATH)
+
+cleanup-runwhen: 
+	helm uninstall ${HELM_RELEASE_NAME} -n ${NAMESPACE}
+
+setup-runwhen-all: setup-sre-stack setup-runwhen
+
+cleanup-runwhen-all: cleanup-runwhen cleanup-sre-stack

codebundles/rds-mysql-conn-count/kill-mysql-sleep-processes.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# MySQL connection details
+# MYSQL_USER="admin"
+# MYSQL_PASSWORD="docdb3421z"
+# MYSQL_HOST="robotshopmysql.cn9m6m4s8zo0.us-west-2.rds.amazonaws.com"
+# PROCESS_USER="shipping"
+echo "MYSQL_USER $MYSQL_USER" 
+echo "MYSQL_PASSWORD $MYSQL_PASSWORD"
+echo "MYSQL_HOST $MYSQL_HOST"
+echo "PROCESS_USER $PROCESS_USER"
+
+# Get process list IDs
+PROCESS_IDS=$(MYSQL_PWD="$MYSQL_PASSWORD" mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -N -s -e "SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER='shipping'")
+
+if [ $? -ne 0 ]; then
+    echo "Error connecting to MySQL"
+    exit 1
+fi
+
+for ID in $PROCESS_IDS; do 
+    MYSQL_PWD="$MYSQL_PASSWORD" mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -e "CALL mysql.rds_kill($ID)"
+    echo "Terminated connection with ID $ID for user 'shipping'"
+done

codebundles/rds-mysql-conn-count/runbook.robot

@@ -0,0 +1,52 @@
+*** Settings ***
+Documentation       This taskset Kills the numbers of sleep process created in MySQL
+Metadata            Author    IFC
+
+Library             BuiltIn
+Library             RW.Core
+Library             RW.platform
+Library             RW.CLI
+
+Suite Setup         Suite Initialization
+
+*** Variables ***
+${MYSQL_PASSWORD_ENV}      %{MYSQL_PASSWORD_ENV}
+
+*** Tasks ***
+Run Bash File
+    [Documentation]    Runs a bash file to kill sleep processes created in MySQL
+    [Tags]    file    script
+    ${rsp}=    RW.CLI.Run Bash File
+    ...    bash_file=kill-mysql-sleep-processes.sh
+    ...    cmd_override=./kill-mysql-sleep-processes.sh
+    ...    env=${env}
+    ...    include_in_history=False
+    RW.Core.Add Pre To Report    Command Stdout:\n${rsp.stdout}
+    RW.Core.Add Pre To Report    Command Stderr:\n${rsp.stderr}
+
+
+*** Keywords ***
+Suite Initialization
+   ${MYSQL_PASSWORD}=    RW.Core.Import Secret    MYSQL_PASSWORD
+    ...    type=string
+    ...    description=MySQL password
+    ...    pattern=\w*
+    ...    example='9jZGIzNDIxego'
+    ${MYSQL_USER}=    RW.Core.Import User Variable    MYSQL_USER
+    ...    type=string
+    ...    description=MySQL Username
+    ...    pattern=\w*
+    ...    example=admin
+    ${MYSQL_HOST}=    RW.Core.Import User Variable    MYSQL_HOST
+    ...    type=string
+    ...    description=MySQL host endpoint
+    ...    pattern=\w*
+    ...    example=robotshopmysql.cn9m6m4s8zo0.us-west-2.rds.amazonaws.com
+    ${PROCESS_USER}=    RW.Core.Import User Variable    PROCESS_USER
+    ...    type=string
+    ...    description=mysql user which created numbers of sleep connections
+    ...    pattern=\w*
+    ...    example=shipping
+
+    Set Suite Variable
+    ...    ${env}    {"MYSQL_USER":"${MYSQL_USER}", "MYSQL_PASSWORD":"${MYSQL_PASSWORD_ENV}", "MYSQL_HOST":"${MYSQL_HOST}", "PROCESS_USER":"${PROCESS_USER}"}

codebundles/rds-mysql-conn-count/runwhen-config/README.md

@@ -0,0 +1 @@
+Placeholder for runwhen platform config.

codebundles/rds-mysql-conn-count/runwhen-config/runbook.yaml

@@ -0,0 +1,26 @@
+location: location-01-us-west1
+codeBundle:
+  repoUrl: https://github.com/infracloudio/infracloud-runwhen-codecollection.git
+  ref: codebundle-rds-mysql-conn
+  pathToRobot: codebundles/rds-mysql-conn-count/runbook.robot
+configProvided:
+  - name: KUBERNETES_DISTRIBUTION_BINARY
+    value: kubectl
+  - name: NAMESPACE
+    value: default
+  - name: ERROR_PATTERN
+    value: (Error|Exception)
+  - name: CONTEXT
+    value: default
+  - name: SERVICE_ERROR_PATTERN
+    value: (Error:)
+  - name: SERVICE_EXCLUDE_PATTERN
+    value: (node_modules|opentelemetry)
+  - name: ANOMALY_THRESHOLD
+    value: '5.0'
+secretsProvided:
+  - name: kubeconfig
+    workspaceKey: kubeconfig
+servicesProvided:
+  - name: kubectl
+    locationServiceName: kubectl-service.shared

codebundles/rds-mysql-conn-count/runwhen-config/sli.yaml

@@ -0,0 +1,34 @@
+displayUnitsLong: OK
+displayUnitsShort: ok
+locations:
+  - location-01-us-west1
+description: >-
+  Watch RDS MySql connection count
+codeBundle:
+  repoUrl: https://github.com/infracloudio/infracloud-runwhen-codecollection.git
+  ref: codebundle-rds-mysql-conn
+  pathToRobot: codebundles/rds-mysql-conn-count/sli.robot
+# read more about intervalStrategy here: https://docs.runwhen.com/public/runwhen-platform/feature-overview/points-on-the-map-slxs/service-level-indicators-slis/interval-strategies
+intervalStrategy: intermezzo
+intervalSeconds: 30
+configProvided:
+  - name: PROMETHEUS_HOSTNAME
+    value: >-
+      http://aeccfb7ff9bfb4705b6218294a7346c3-2081802229.us-west-2.elb.amazonaws.com/prometheus/api/v1
+  - name: QUERY
+    value: >-
+      aws_rds_database_connections_average{dimension_DBInstanceIdentifier="robotshopmysql"} > 1
+  - name: TRANSFORM
+    value: RAW
+  - name: STEP
+    value: '30'
+  - name: DATA_COLUMN
+    value: '1'
+  - name: NO_RESULT_OVERWRITE
+    value: 'Yes'
+  - name: NO_RESULT_VALUE
+    value: '0'
+secretsProvided: []
+servicesProvided:
+  - name: curl
+    locationServiceName: curl-service.shared

codebundles/rds-mysql-conn-count/runwhen-config/slo.yaml

@@ -0,0 +1,8 @@
+codeBundle:
+  repoUrl: https://github.com/runwhen-contrib/rw-public-codecollection
+  pathToYaml: codebundles/slo-default/queries.yaml
+  ref: main
+sloSpecType: simple-mwmb
+objective: 95
+threshold: 48
+operand: lt

codebundles/rds-mysql-conn-count/runwhen-config/slx.yaml

@@ -0,0 +1,9 @@
+statement: RDS MySql connections should be within 80% of total max connection.
+alias: RDS MySql Connections Count
+metricType: gauge
+asMeasuredBy: Score based on promethues query
+icon: Cloud
+owners:
+  - [email protected]
+imageURL: >-
+  https://storage.googleapis.com/runwhen-nonprod-shared-images/icons/kubernetes/resources/labeled/ns.svg

codebundles/rds-mysql-conn-count/sli.robot

@@ -0,0 +1,87 @@
+*** Settings ***
+Metadata          Author    InfraCloud
+Documentation     Run a PromQL query against Prometheus instant query API, perform a provided transform, and return the result.
+Force Tags        Prometheus    Prom    PromQL    Query    Metric    Aggregate
+Suite Setup       Suite Initialization
+Library           RW.Core
+Library           RW.Prometheus
+
+*** Variables ***
+${ENV_PROMETHEUS_HOST}      %{ENV_PROMETHEUS_HOST}
+${ENV_QUERY}      %{ENV_QUERY}
+
+*** Keywords ***
+Suite Initialization
+    ${CURL_SERVICE}=    RW.Core.Import Service    curl
+    ...    type=string
+    ...    description=The selected RunWhen Service to use for accessing services within a network.
+    ...    pattern=\w*
+    ...    example=curl-service.shared
+    ...    default=curl-service.shared
+    ${OPTIONAL_HEADERS}=    RW.Core.Import Secret    OPTIONAL_HEADERS
+    ...    type=string
+    ...    description=A json string of headers to include in the request against the Prometheus instance. This can include your token.
+    ...    pattern=\w*
+    ...    default="{"my-header":"my-value"}"
+    ...    example='{"my-header":"my-value", "Authorization": "Bearer mytoken"}'
+    RW.Core.Import User Variable    PROMETHEUS_HOSTNAME
+    ...    type=string
+    ...    description=The prometheus endpoint to perform requests against.
+    ...    pattern=\w*
+    ...    example=https://myprometheus/api/v1/
+    RW.Core.Import User Variable    QUERY
+    ...    type=string
+    ...    description=The PromQL statement used to query metrics.
+    ...    pattern=\w*
+    ...    example=sysdig_container_cpu_quota_used_percent > 75 or sysdig_container_memory_limit_used_percent> 75
+    ...    default=sysdig_container_cpu_quota_used_percent > 75 or sysdig_container_memory_limit_used_percent> 75
+    RW.Core.Import User Variable    TRANSFORM
+    ...    type=string
+    ...    enum=[Raw,Max,Average,Minimum,Sum,First,Last]
+    ...    description=What transform method to apply to the column data. First and Last are position relative, so Last is the most recent value. Use Raw to skip transform. 
+    ...    default=Last
+    ...    example=Last
+    RW.Core.Import User Variable    STEP
+    ...    type=string
+    ...    description=The step interval in seconds requested from the Prometheus API.
+    ...    pattern="^[0-9]*$"
+    ...    default=30
+    ...    example=30
+    RW.Core.Import User Variable    DATA_COLUMN
+    ...    type=string
+    ...    description=Which column of the result data to perform aggregation on. Typically 0 is the timestamp, whereas 1 is the metric value.
+    ...    pattern="^[0-9]*$"
+    ...    default=1
+    ...    example=1
+    RW.Core.Import User Variable    NO_RESULT_OVERWRITE
+    ...    type=string
+    ...    description=Determine how to handle queries with no result data. Set to Yes to write a metric (specified below) or No to accept the null result.
+    ...    pattern=\w*
+    ...    enum=[Yes,No]
+    ...    default=Yes
+    RW.Core.Import User Variable    NO_RESULT_VALUE
+    ...    type=string
+    ...    description=Set the metric value that should be stored when no data result is available.
+    ...    pattern=\d*
+    ...    default=0
+    ...    example=0
+    Set Suite Variable    ${CURL_SERVICE}    ${CURL_SERVICE}
+    Set Suite Variable    ${OPTIONAL_HEADERS}    ${OPTIONAL_HEADERS}
+    Set Suite Variable    ${NO_RESULT_OVERWRITE}    ${NO_RESULT_OVERWRITE}
+    Set Suite Variable    ${NO_RESULT_VALUE}    ${NO_RESULT_VALUE}
+
+*** Tasks ***
+Querying Prometheus Instance And Pushing Aggregated Data
+    Log      ${ENV_QUERY}
+    ${rsp}=    RW.Prometheus.Query Instant
+    ...    api_url=${ENV_PROMETHEUS_HOST}
+    ...    query=${ENV_QUERY}
+    ...    step=${STEP}
+    ...    target_service=${CURL_SERVICE}
+    ${data}=    Set Variable    ${rsp["data"]}
+    ${metric}=    RW.Prometheus.Transform Data
+    ...    data=${data}
+    ...    method=${TRANSFORM}
+    ...    no_result_overwrite=${NO_RESULT_OVERWRITE}
+    ...    no_result_value=${NO_RESULT_VALUE}
+    RW.Core.Push Metric    ${metric}

codebundles/rds-mysql-conn-count/test/manifests/sli-deployment.yaml

@@ -0,0 +1,38 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: rds-mysql-connection-count
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: rds-mysql-connection-count
+  template:
+    metadata:
+      labels:
+        app: rds-mysql-connection-count
+    spec:
+      containers:
+      - name: rds-mysql-connection-count
+        image: 590183940259.dkr.ecr.us-west-2.amazonaws.com/runwhen:latest
+        command:
+          - "bash"
+          - "-c"
+          - "ro /app/codecollection/codebundles/rds-mysql-conn-count/runbook.robot && while true; do sleep 5; done"
+        ports:
+        - containerPort: 3000
+        env:
+        - name: ENV_PROMETHEUS_HOST
+          value: "http://ab916f39fadce498ead455d91e808053-1900228415.us-west-2.elb.amazonaws.com/prometheus/api/v1"
+        - name: ENV_QUERY
+          value: "aws_rds_database_connections_average{dimension_DBInstanceIdentifier=\"robotshopmysql\"} > 1"
+        - name: MYSQL_USER
+          value: "admin"
+        - name: MYSQL_PASSWORD_ENV
+          value: "docdb3421z"
+        - name: MYSQL_HOST
+          value: "robotshopmysql.c5eo4uy8mys1.us-west-2.rds.amazonaws.com"
+        - name: PROCESS_USER
+          value: "shipping"
+        - name: RW_PATH_TO_ROBOT
+          value: "/app/codecollection/codebundles/rds-mysql-conn-count/runbook.robot"

codebundles/rds-mysql-conn-count/test/scripts/create-msql-sleeping-connections.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# MySQL connection details
+MYSQL_USER="shipping"
+MYSQL_PASSWORD="secret"
+MYSQL_HOST="robotshopmysql.c5eo4uy8mys1.us-west-2.rds.amazonaws.com"
+
+CONNECTIONS=30
+SLEEP_TIMEOUT=260
+
+for ((i=1;i<=$CONNECTIONS;i++))
+do
+    MYSQL_PWD="$MYSQL_PASSWORD" mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -N -s -e "SELECT SLEEP(${SLEEP_TIMEOUT});" &
+done