px-bench-main.yml

apiVersion: v1
kind: ConfigMap
metadata:
  name: files
  namespace: px-bench
data:
  run.sh: |-
    #!/usr/bin/bash
    mkdir /tmp/fio
    #iterate throught the block sizes and rw settings
    i=0
    for sc in $storageclass_list; do
        for blocksize in ${blocksize_list}; do
            for readwrite in ${readwrite_list}; do
                export sc readwrite blocksize
                export loops=1000
                export time_based=time_based
                i=$[$i+1]
                filename=/tmp/fio/$(printf %.3d $i)_${sc}_$blocksize-$readwrite.fio
                envsubst < /px-bench/main.fio.tmpl > $filename
                if [[ ! " ${mixed_workloads[@]} " =~ " ${rw} " ]]; then
                    sed -i '/rwmix/d' $filename
                fi
                sed -i '/##/d' $filename
            done
        done
    done
    for sc in $storageclass_list; do
        filename=/tmp/fio/000_${sc}_warmup.fio
        export blocksize=$warmup_blocksize
        export readwrite=write
        export loops=$warmup_loops
        export runtime=86400
        export time_based=
        envsubst < /px-bench/main.fio.tmpl > $filename
    done
    nodeSelector=$(echo $node_selector | sed 's/=/: /')
    kubectl create configmap fio-config --from-file=/tmp/fio -n px-bench
    for sc in $storageclass_list; do
      volumes="$volumes\
          - name: mnt-$sc
            persistentVolumeClaim:
              claimName: mnt-$sc
    "
      volumemounts="$volumemounts\
            - name: mnt-$sc
              mountPath: /mnt/$sc
    "
      kubectl apply -f - <<EOF
    kind: PersistentVolumeClaim
    apiVersion: v1
    metadata:
       name: mnt-$sc
       namespace: px-bench
    spec:
       storageClassName: $sc
       accessModes:
         - ReadWriteOnce
       resources:
         requests:
           storage: $pvc_size
    EOF
    done
    kubectl apply -f - <<EOF
    apiVersion: batch/v1
    kind: Job
    metadata:
      name: fio
      namespace: px-bench
    spec:
      template:
        metadata:
          name: fio
          labels:
            px-bench: fio
        spec:
          nodeSelector:
            $nodeSelector
          securityContext:
            fsGroupChangePolicy: "OnRootMismatch"
          serviceAccountName: px-bench
          terminationGracePeriodSeconds: 0
          containers:
          - name: fio
            image: andrewh1978/px-bench
            imagePullPolicy: "Always"
            command: [ "/bin/bash", "/px-bench/fio.sh" ]
            securityContext:
              allowPrivilegeEscalation: false
              seccompProfile:
                type: RuntimeDefault
              capabilities:
                drop: ["ALL"]
            volumeMounts:
            - name: files
              mountPath: /px-bench
            - name: fio
              mountPath: /fio
    $volumemounts
          volumes:
          - name: files
            configMap:
              name: files
          - name: fio
            configMap:
              name: fio-config
    $volumes
          restartPolicy: OnFailure
    EOF
    kubectl wait --for=condition=complete --timeout=86400s job/fio -n px-bench
  main.fio.tmpl: |-
    ############################################################
    ## main.fio.tmpl
    ##
    ## This is a template for creating a fio jobfile.
    ## Do not edit. This is read by other scripts/functions.
    ##
    ############################################################
    # This file was automatically created by px-bench
    ############################################################
    #
    [global]
    size=$file_size
    runtime=${runtime}
    ${time_based}
    loops=${loops}
    iodepth=${iodepth}
    ioengine=${ioengine}
    direct=${direct}
    fsync_on_close=1
    numjobs=${numjobs}
    group_reporting=1
    directory=/mnt/${sc}
    filename=fio.dat
    
    ##----layout output-------
    [layout-output]
    blocksize=${blocksize}
    readwrite=read  # This will force fio to write to create the file, not just sparse creation.
    create_only=1
    
    ##----Begin Workload Test-------
    [${sc}_${blocksize}_${readwrite}]
    stonewall
    blocksize=${blocksize}
    readwrite=${readwrite}
    rwmixread=${rwmixread}
    rwmixwrite=${rwmixwrite}
    ramp_time=$ramp_time
  fio.sh: |-
    #!/usr/bin/bash
    mkdir /tmp/output-json
    mkdir /tmp/output-csv
    for i in /fio/*; do
      echo $(date +"%Y-%m-%d %H:%M:%S") - Running fio for $i
      fio $i --output-format=json | tee /tmp/output-json/$(basename $i .fio)
      if ! [[ $i =~ warmup ]] ; then
        jq '.jobs[1] | {jobname, read_iops: .read.iops, write_iops: .write.iops, read_lat: .read.lat_ns.mean, write_lat: .write.lat_ns.mean, read_bw: .read.bw, write_bw: .write.bw }' /tmp/output-json/$(basename $i .fio) | jq -r ' flatten | @csv' >/tmp/output-csv/$(basename $i .fio)
      fi
    done
    kubectl create configmap fio-output --from-file=/tmp/output-json -n px-bench
    kubectl create configmap fio-csv --from-file=/tmp/output-csv -n px-bench
    echo "storageclass, blocksize, workload, read_iops, write_iops, read_lat, write_lat, read_bw, write_bw" >/tmp/csv
    kubectl get cm -n px-bench fio-csv -o yaml | grep -A 1 '|' | grep -v '|' | sed 's/^ *//;s/"//g;s/_/,/g' >>/tmp/csv
    kubectl create configmap csv --from-file=/tmp/csv -n px-bench
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: px-bench
  namespace: px-bench
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: px-bench
  namespace: px-bench
rules:
  - apiGroups: [""]
    resources:
      - configmaps
      - persistentvolumeclaims
    verbs:
      - create
      - get
  - apiGroups: ["batch"]
    resources:
      - jobs
    verbs:
      - create
      - list
      - get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: px-bench
  namespace: px-bench
subjects:
  - kind: ServiceAccount
    name: px-bench
roleRef:
  kind: Role
  name: px-bench
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: batch/v1
kind: Job
metadata:
  name: run
  namespace: px-bench
spec:
  template:
    metadata:
      name: run
      labels:
        px-bench: run
    spec:
      securityContext:
        fsGroupChangePolicy: "OnRootMismatch"
      serviceAccountName: px-bench
      terminationGracePeriodSeconds: 0
      containers:
      - name: run
        image: andrewh1978/px-bench
        imagePullPolicy: "Always"
        command: [ "/bin/bash", "/px-bench/run.sh" ]
        securityContext:
          allowPrivilegeEscalation: false
          seccompProfile:
            type: RuntimeDefault
          capabilities:
            drop: ["ALL"]
        envFrom:
        - configMapRef:
            name: env
        volumeMounts:
        - name: files
          mountPath: /px-bench
      volumes:
      - name: files
        configMap:
          name: files
      restartPolicy: OnFailure