Skip to content

Commit

Permalink
add allocated and used disk size metrics
Browse files Browse the repository at this point in the history 
add allocated and used disk size metrics
Signed-off-by: avlitman <[email protected]>
  • Loading branch information
@avlitman
avlitman committed Nov 5, 2024
1 parent 74ba74e commit 10fb271
Show file tree
Hide file tree
Showing 7 changed files with 245 additions and 51 deletions.
3 changes: 3 additions & 0 deletions docs/observability/metrics.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,9 @@ The total number of VMs created by namespace and virt-api pod, since install. Ty
### kubevirt_vm_created_total
The total number of VMs created by namespace, since install. Type: Counter.

### kubevirt_vm_disk_allocated_size_bytes
Allocated disk size of a Virtual Machine in bytes, based on its PersistentVolumeClaim. Includes persistentvolumeclaim (PVC name), volume_mode (disk presentation mode: Filesystem or Block), and device (disk name). Type: Gauge.

### kubevirt_vm_error_status_last_transition_timestamp_seconds
Virtual Machine last transition timestamp to error status. Type: Counter.

Expand Down
19 changes: 11 additions & 8 deletions pkg/monitoring/metrics/virt-controller/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,19 +41,21 @@ var (
vmSnapshotMetrics,
}

vmInformer cache.SharedIndexInformer
vmiInformer cache.SharedIndexInformer
clusterInstanceTypeInformer cache.SharedIndexInformer
instanceTypeInformer cache.SharedIndexInformer
clusterPreferenceInformer cache.SharedIndexInformer
preferenceInformer cache.SharedIndexInformer
vmiMigrationInformer cache.SharedIndexInformer
clusterConfig *virtconfig.ClusterConfig
vmInformer cache.SharedIndexInformer
vmiInformer cache.SharedIndexInformer
persistentVolumeClaimInformer cache.SharedIndexInformer
clusterInstanceTypeInformer cache.SharedIndexInformer
instanceTypeInformer cache.SharedIndexInformer
clusterPreferenceInformer cache.SharedIndexInformer
preferenceInformer cache.SharedIndexInformer
vmiMigrationInformer cache.SharedIndexInformer
clusterConfig *virtconfig.ClusterConfig
)

func SetupMetrics(
vm cache.SharedIndexInformer,
vmi cache.SharedIndexInformer,
pvc cache.SharedIndexInformer,
clusterInstanceType cache.SharedIndexInformer,
instanceType cache.SharedIndexInformer,
clusterPreference cache.SharedIndexInformer,
Expand All @@ -63,6 +65,7 @@ func SetupMetrics(
) error {
vmInformer = vm
vmiInformer = vmi
persistentVolumeClaimInformer = pvc
clusterInstanceTypeInformer = clusterInstanceType
instanceTypeInformer = instanceType
clusterPreferenceInformer = clusterPreference
Expand Down
148 changes: 107 additions & 41 deletions pkg/monitoring/metrics/virt-controller/vmstats_collector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,57 +21,21 @@ package virt_controller

import (
"github.com/machadovilaca/operator-observability/pkg/operatormetrics"
k8sv1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"kubevirt.io/client-go/log"

k6tv1 "kubevirt.io/api/core/v1"
"kubevirt.io/client-go/log"

"kubevirt.io/kubevirt/pkg/controller"
)

var (
vmStatsCollector = operatormetrics.Collector{
Metrics: append(timestampMetrics, vmResourceRequests, vmResourceLimits, vmInfo),
Metrics: append(timestampMetrics, vmResourceRequests, vmResourceLimits, vmInfo, vmDiskAllocatedSize),
CollectCallback: vmStatsCollectorCallback,
}

vmInfo = operatormetrics.NewGaugeVec(
operatormetrics.MetricOpts{
Name: "kubevirt_vm_info",
Help: "Information about Virtual Machines.",
},
[]string{
// Basic info
"name", "namespace",

// VM annotations
"os", "workload", "flavor",

// VM Machine Type
"machine_type",

// Instance type
"instance_type", "preference",

// Status
"status", "status_group",
},
)

vmResourceRequests = operatormetrics.NewGaugeVec(
operatormetrics.MetricOpts{
Name: "kubevirt_vm_resource_requests",
Help: "Resources requested by Virtual Machine. Reports memory and CPU requests.",
},
[]string{"name", "namespace", "resource", "unit"},
)

vmResourceLimits = operatormetrics.NewGaugeVec(
operatormetrics.MetricOpts{
Name: "kubevirt_vm_resource_limits",
Help: "Resources limits by Virtual Machine. Reports memory and CPU limits.",
},
[]string{"name", "namespace", "resource", "unit"},
)

timestampMetrics = []operatormetrics.Metric{
startingTimestamp,
runningTimestamp,
Expand Down Expand Up @@ -152,6 +116,55 @@ var (
k6tv1.VirtualMachineStatusPvcNotFound,
k6tv1.VirtualMachineStatusDataVolumeError,
}

vmResourceRequests = operatormetrics.NewGaugeVec(
operatormetrics.MetricOpts{
Name: "kubevirt_vm_resource_requests",
Help: "Resources requested by Virtual Machine. Reports memory and CPU requests.",
},
[]string{"name", "namespace", "resource", "unit"},
)

vmResourceLimits = operatormetrics.NewGaugeVec(
operatormetrics.MetricOpts{
Name: "kubevirt_vm_resource_limits",
Help: "Resources limits by Virtual Machine. Reports memory and CPU limits.",
},
[]string{"name", "namespace", "resource", "unit"},
)

vmInfo = operatormetrics.NewGaugeVec(
operatormetrics.MetricOpts{
Name: "kubevirt_vm_info",
Help: "Information about Virtual Machines.",
},
[]string{
// Basic info
"name", "namespace",

// VM annotations
"os", "workload", "flavor",

// VM Machine Type
"machine_type",

// Instance type
"instance_type", "preference",

// Status
"status", "status_group",
},
)

vmDiskAllocatedSize = operatormetrics.NewGaugeVec(
operatormetrics.MetricOpts{
Name: "kubevirt_vm_disk_allocated_size_bytes",
Help: "Allocated disk size of a Virtual Machine in bytes, based on its PersistentVolumeClaim. " +
"Includes persistentvolumeclaim (PVC name), volume_mode (disk presentation mode: Filesystem or Block), " +
"and device (disk name).",
},
[]string{"name", "namespace", "persistentvolumeclaim", "volume_mode", "device"},
)
)

func vmStatsCollectorCallback() []operatormetrics.CollectorResult {
Expand All @@ -168,6 +181,7 @@ func vmStatsCollectorCallback() []operatormetrics.CollectorResult {
}

var results []operatormetrics.CollectorResult
results = append(results, CollectDiskAllocatedSize(vms)...)
results = append(results, CollectVMsInfo(vms)...)
results = append(results, CollectResourceRequestsAndLimits(vms)...)
results = append(results, reportVmsStats(vms)...)
Expand Down Expand Up @@ -467,3 +481,55 @@ func containsCondition(target k6tv1.VirtualMachineConditionType, elems []k6tv1.V
}
return false
}

func CollectDiskAllocatedSize(vms []*k6tv1.VirtualMachine) []operatormetrics.CollectorResult {
var cr []operatormetrics.CollectorResult

for _, vm := range vms {
if vm.Spec.Template != nil {
cr = append(cr, collectDiskMetricsFromPVC(vm)...)
}
}

return cr
}

func collectDiskMetricsFromPVC(vm *k6tv1.VirtualMachine) []operatormetrics.CollectorResult {
var cr []operatormetrics.CollectorResult

for _, vol := range vm.Spec.Template.Spec.Volumes {
if vol.PersistentVolumeClaim != nil {
diskName := vol.Name
pvcName := vol.PersistentVolumeClaim.ClaimName

key := controller.NamespacedKey(vm.Namespace, pvcName)
obj, exists, err := persistentVolumeClaimInformer.GetStore().GetByKey(key)
if err != nil {
log.Log.Errorf("Error retrieving PVC %s in namespace %s: %v", pvcName, vm.Namespace, err)
continue
}
if !exists {
log.Log.Warningf("PVC %s in namespace %s does not exist", pvcName, vm.Namespace)
continue
}
pvc, ok := obj.(*k8sv1.PersistentVolumeClaim)
if !ok {
log.Log.Warningf("Object for PVC %s in namespace %s is not of expected type", pvcName, vm.Namespace)
continue
}

pvcSize := pvc.Spec.Resources.Requests.Storage()
volumeMode := "null"
if pvc.Spec.VolumeMode != nil {
volumeMode = string(*pvc.Spec.VolumeMode)
}
cr = append(cr, operatormetrics.CollectorResult{
Metric: vmDiskAllocatedSize,
Value: float64(pvcSize.Value()),
Labels: []string{vm.Name, vm.Namespace, pvcName, volumeMode, diskName},
})
}
}

return cr
}
103 changes: 103 additions & 0 deletions pkg/monitoring/metrics/virt-controller/vmstats_collector_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

k6tv1 "kubevirt.io/api/core/v1"

"kubevirt.io/kubevirt/pkg/pointer"
"kubevirt.io/kubevirt/pkg/testutils"
)

var _ = Describe("VM Stats Collector", func() {
Expand Down Expand Up @@ -368,4 +371,104 @@ var _ = Describe("VM Stats Collector", func() {
Expect(crs[2].Labels).To(Equal([]string{"testvm", "test-ns", "cpu", "sockets"}))
})
})

Context("PVC allocated size metric collection", func() {
BeforeEach(func() {
persistentVolumeClaimInformer, _ = testutils.NewFakeInformerFor(&k8sv1.PersistentVolumeClaim{})
})

createPVC := func(namespace, name string, size resource.Quantity, volumeMode *k8sv1.PersistentVolumeMode) *k8sv1.PersistentVolumeClaim {
storageClassName := "rook-ceph-block"

return &k8sv1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name},
Spec: k8sv1.PersistentVolumeClaimSpec{
AccessModes: []k8sv1.PersistentVolumeAccessMode{k8sv1.ReadWriteOnce},
VolumeMode: volumeMode,
Resources: k8sv1.VolumeResourceRequirements{
Requests: k8sv1.ResourceList{
k8sv1.ResourceStorage: size,
},
},
StorageClassName: &storageClassName,
},
}
}

It("should collect PVC size metrics correctly", func() {
pvc := createPVC("default", "test-vm-pvc", resource.MustParse("5Gi"), pointer.P(k8sv1.PersistentVolumeFilesystem))
err := persistentVolumeClaimInformer.GetIndexer().Add(pvc)
Expect(err).ToNot(HaveOccurred())

vm := &k6tv1.VirtualMachine{
ObjectMeta: metav1.ObjectMeta{
Namespace: "default",
Name: "test-vm",
},
Spec: k6tv1.VirtualMachineSpec{
Template: &k6tv1.VirtualMachineInstanceTemplateSpec{
Spec: k6tv1.VirtualMachineInstanceSpec{
Volumes: []k6tv1.Volume{
{
Name: "rootdisk",
VolumeSource: k6tv1.VolumeSource{
PersistentVolumeClaim: &k6tv1.PersistentVolumeClaimVolumeSource{
PersistentVolumeClaimVolumeSource: k8sv1.PersistentVolumeClaimVolumeSource{
ClaimName: "test-vm-pvc",
},
},
},
},
},
},
},
},
}

results := CollectDiskAllocatedSize([]*k6tv1.VirtualMachine{vm})

Expect(results).ToNot(BeEmpty())
Expect(results[0].Metric.GetOpts().Name).To(Equal("kubevirt_vm_disk_allocated_size_bytes"))
Expect(results[0].Value).To(Equal(float64(5 * 1024 * 1024 * 1024)))
Expect(results[0].Labels).To(Equal([]string{"test-vm", "default", "test-vm-pvc", "Filesystem", "rootdisk"}))
})

It("should handle PVC with nil volume mode", func() {
pvc := createPVC("default", "test-vm-pvc-nil-mode", resource.MustParse("3Gi"), nil)
err := persistentVolumeClaimInformer.GetIndexer().Add(pvc)
Expect(err).ToNot(HaveOccurred())

vm := &k6tv1.VirtualMachine{
ObjectMeta: metav1.ObjectMeta{
Namespace: "default",
Name: "test-vm-nil-mode",
},
Spec: k6tv1.VirtualMachineSpec{
Template: &k6tv1.VirtualMachineInstanceTemplateSpec{
Spec: k6tv1.VirtualMachineInstanceSpec{
Volumes: []k6tv1.Volume{
{
Name: "rootdisk",
VolumeSource: k6tv1.VolumeSource{
PersistentVolumeClaim: &k6tv1.PersistentVolumeClaimVolumeSource{
PersistentVolumeClaimVolumeSource: k8sv1.PersistentVolumeClaimVolumeSource{
ClaimName: "test-vm-pvc-nil-mode",
},
},
},
},
},
},
},
},
}

results := CollectDiskAllocatedSize([]*k6tv1.VirtualMachine{vm})

Expect(results).ToNot(BeEmpty())
Expect(results[0].Metric.GetOpts().Name).To(Equal("kubevirt_vm_disk_allocated_size_bytes"))
Expect(results[0].Value).To(Equal(float64(3 * 1024 * 1024 * 1024)))
Expect(results[0].Labels).To(Equal([]string{"test-vm-nil-mode", "default", "test-vm-pvc-nil-mode", "null", "rootdisk"}))
})
})
})
1 change: 1 addition & 0 deletions pkg/virt-controller/watch/application.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -435,6 +435,7 @@ func Execute() {
if err := metrics.SetupMetrics(
app.vmInformer,
app.vmiInformer,
app.persistentVolumeClaimInformer,
app.clusterInstancetypeInformer,
app.instancetypeInformer,
app.clusterPreferenceInformer,
Expand Down
Loading

0 comments on commit 10fb271

Please sign in to comment.