diff --git a/pkg/operator/ceph/controller/spec.go b/pkg/operator/ceph/controller/spec.go index 9a2afa27b78c..73c5a424ef68 100644 --- a/pkg/operator/ceph/controller/spec.go +++ b/pkg/operator/ceph/controller/spec.go @@ -757,9 +757,7 @@ func PrivilegedContext(runAsRoot bool) *v1.SecurityContext { return sec } -// LogCollectorContainer rotate logs -func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec) *v1.Container { - +func GetLogRotateConfig(c cephv1.ClusterSpec) (resource.Quantity, string) { var maxLogSize resource.Quantity if c.LogCollector.MaxLogSize != nil { size := c.LogCollector.MaxLogSize.Value() / 1000 / 1000 @@ -771,20 +769,28 @@ func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec) *v1.Contai maxLogSize = resource.MustParse(fmt.Sprintf("%dM", size)) } - rotation := "7" - if strings.Contains(daemonID, "-client.rbd-mirror") { - rotation = "28" - } - var periodicity string - if c.LogCollector.Periodicity == "1h" || c.LogCollector.Periodicity == "hourly" { + switch c.LogCollector.Periodicity { + case "1h", "hourly": periodicity = "hourly" - } else if c.LogCollector.Periodicity == "weekly" || c.LogCollector.Periodicity == "monthly" { + case "weekly", "monthly": periodicity = c.LogCollector.Periodicity - } else { + default: periodicity = "daily" } + return maxLogSize, periodicity +} + +// LogCollectorContainer rotate logs +func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec) *v1.Container { + maxLogSize, periodicity := GetLogRotateConfig(c) + + rotation := "7" + if strings.Contains(daemonID, "-client.rbd-mirror") { + rotation = "28" + } + logger.Debugf("setting periodicity to %q. Supported periodicity are hourly, daily, weekly and monthly", periodicity) return &v1.Container{ diff --git a/pkg/operator/ceph/csi/controller.go b/pkg/operator/ceph/csi/controller.go index 87070a2a4bc2..467791a11707 100644 --- a/pkg/operator/ceph/csi/controller.go +++ b/pkg/operator/ceph/csi/controller.go @@ -205,7 +205,7 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to list ceph clusters") } - // // Do not nothing if no ceph cluster is present + // Do nothing if no ceph cluster is present if len(cephClusters.Items) == 0 { logger.Debug("no ceph cluster found not deploying ceph csi driver") EnableRBD, EnableCephFS, EnableNFS = false, false, false @@ -217,6 +217,10 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e return reconcile.Result{}, nil } + // if at least one cephcluster is present update the csi lograte sidecar + // with the first listed ceph cluster specs with logrotate enabled + setCSILogrotateParams(cephClusters.Items) + err = peermap.CreateOrUpdateConfig(r.opManagerContext, r.context, &peermap.PeerIDMappings{}) if err != nil { return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to create pool ID mapping config map") @@ -299,3 +303,24 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e return reconcileResult, nil } + +func setCSILogrotateParams(cephClustersItems []cephv1.CephCluster) { + logger.Debug("set logrotate values in csi param") + spec := cephClustersItems[0].Spec + for _, cluster := range cephClustersItems { + if cluster.Spec.LogCollector.Enabled { + spec = cluster.Spec + break + } + } + CSIParam.CsiLogDirPath = spec.DataDirHostPath + if spec.DataDirHostPath == "" { + CSIParam.CsiLogDirPath = k8sutil.DataDir + } + CSIParam.CSILogRotation = spec.LogCollector.Enabled + if spec.LogCollector.Enabled { + maxSize, period := opcontroller.GetLogRotateConfig(spec) + CSIParam.CSILogRotationMaxSize = maxSize.String() + CSIParam.CSILogRotationPeriod = period + } +} diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 70bf0c6f4374..a02a98338b3d 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -52,6 +52,7 @@ type Param struct { ResizerImage string DriverNamePrefix string KubeletDirPath string + CsiLogDirPath string ForceCephFSKernelClient string CephFSKernelMountOptions string CephFSPluginUpdateStrategy string @@ -97,6 +98,10 @@ type Param struct { CSICephFSPodLabels map[string]string CSINFSPodLabels map[string]string CSIRBDPodLabels map[string]string + CSILogRotation bool + CSILogRotationMaxSize string + CSILogRotationPeriod string + CSILogFolder string } type templateParam struct { @@ -178,6 +183,9 @@ var ( //go:embed template/nfs/csi-nfsplugin-holder.yaml NFSPluginHolderTemplatePath string + //go:embed template/csi-logrotate-sidecar.yaml + LogrotateTemplatePath string + holderEnabled bool ) @@ -349,11 +357,19 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI if err != nil { return errors.Wrap(err, "failed to load rbdplugin template") } + if tp.CSILogRotation { + tp.CSILogFolder = "rbd-plugin" + applyLogrotateSidecar(&rbdPlugin.Spec.Template, "csi-rbd-daemonset-log-collector", LogrotateTemplatePath, tp) + } rbdProvisionerDeployment, err = templateToDeployment("rbd-provisioner", RBDProvisionerDepTemplatePath, tp) if err != nil { return errors.Wrap(err, "failed to load rbd provisioner deployment template") } + if tp.CSILogRotation { + tp.CSILogFolder = "rbd-provisioner" + applyLogrotateSidecar(&rbdProvisionerDeployment.Spec.Template, "csi-rbd-deployment-log-collector", LogrotateTemplatePath, tp) + } // Create service if either liveness or GRPC metrics are enabled. if CSIParam.EnableLiveness { diff --git a/pkg/operator/ceph/csi/template/csi-logrotate-sidecar.yaml b/pkg/operator/ceph/csi/template/csi-logrotate-sidecar.yaml new file mode 100644 index 000000000000..63a47286cedc --- /dev/null +++ b/pkg/operator/ceph/csi/template/csi-logrotate-sidecar.yaml @@ -0,0 +1,35 @@ +args: + - | + echo "Starting the csi-logrotate-sidecar" + mkdir -p {{ .CsiLogDirPath }}/cephcsi/logrotate-config/{{ .CSILogFolder }}; + echo '{{ .CsiLogDirPath }}cephcsi/log/{{ .CSILogFolder }}/*.log { + {{ .CSILogRotationPeriod }} + missingok + rotate 7 + compress + copytruncate + notifempty + }' > {{ .CsiLogDirPath }}/cephcsi/logrotate-config/{{ .CSILogFolder }}/csi; + echo "File creation container completed"; + + LOG_ROTATE_CEPH_CSI_FILE={{ .CsiLogDirPath }}/cephcsi/logrotate-config/{{ .CSILogFolder }}/csi + LOG_MAX_SIZE={{ .CSILogRotationMaxSize }} + if [ "$LOG_MAX_SIZE" != "0" ]; then + sed --in-place "4i \ \ \ \ maxsize $LOG_MAX_SIZE" "$LOG_ROTATE_CEPH_CSI_FILE" + fi + + while true; do + logrotate --verbose "$LOG_ROTATE_CEPH_CSI_FILE" + sleep 15m + done +command: + - /bin/sh + - -c +image: {{ .CSIPluginImage }} +imagePullPolicy: IfNotPresent +name: log-collector +volumeMounts: + - mountPath: {{ .CsiLogDirPath }}/cephcsi/logrotate-config/{{ .CSILogFolder }} + name: csi-logs-logrotate + - mountPath: {{ .CsiLogDirPath }}/cephcsi/log/{{ .CSILogFolder }} + name: csi-log diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml index 05dc8bfc443c..2265b0182136 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml @@ -38,9 +38,9 @@ spec: - "--extra-create-metadata=true" - "--prevent-volume-mode-conversion=true" - "--feature-gates=HonorPVReclaimPolicy=true" - {{- if .EnableCSITopology }} + {{ if .EnableCSITopology }} - "--feature-gates=Topology=true" - {{- end }} + {{ end }} env: - name: ADDRESS value: unix:///csi/csi-provisioner.sock @@ -116,7 +116,7 @@ spec: {{ if .EnableOMAPGenerator }} - name: csi-omap-generator image: {{ .CSIPluginImage }} - args : + args: - "--type=controller" - "--drivernamespace=$(DRIVER_NAMESPACE)" - "--v={{ .LogLevel }}" @@ -142,7 +142,7 @@ spec: {{ if .EnableCSIAddonsSideCar }} - name: csi-addons image: {{ .CSIAddonsImage }} - args : + args: - "--node-id=$(NODE_ID)" - "--v={{ .LogLevel }}" - "--csi-addons-address=$(CSIADDONS_ENDPOINT)" @@ -155,6 +155,11 @@ spec: - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" + {{ if .CSILogRotation }} + - "--logtostderr=false" + - "--alsologtostderr=true" + - "--log_file={{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner/csi-addons.log" + {{ end }} ports: - containerPort: {{ .CSIAddonsPort }} env: @@ -180,10 +185,14 @@ spec: volumeMounts: - name: socket-dir mountPath: /csi + {{ if .CSILogRotation }} + - mountPath: {{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner + name: csi-log + {{ end }} {{ end }} - name: csi-rbdplugin image: {{ .CSIPluginImage }} - args : + args: - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--v={{ .LogLevel }}" @@ -191,6 +200,11 @@ spec: - "--controllerserver=true" - "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com" - "--pidlimit=-1" + {{ if .CSILogRotation }} + - "--logtostderr=false" + - "--alsologtostderr=true" + - "--log_file={{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner/csi-rbdplugin.log" + {{ end }} {{ if .EnableCSIAddonsSideCar }} - "--csi-addons-endpoint=$(CSIADDONS_ENDPOINT)" {{ end }} @@ -225,6 +239,10 @@ spec: mountPath: /csi - mountPath: /dev name: host-dev + {{ if .CSILogRotation }} + - mountPath: {{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner + name: csi-log + {{ end }} - mountPath: /sys name: host-sys - mountPath: /lib/modules @@ -272,6 +290,13 @@ spec: - name: host-dev hostPath: path: /dev + - name: csi-log + hostPath: + path: {{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner + type: DirectoryOrCreate + - name: csi-logs-logrotate + emptyDir: + type: DirectoryOrCreate - name: host-sys hostPath: path: /sys diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml index 960598ace7a8..733cfa2ddd87 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml @@ -73,6 +73,11 @@ spec: - "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com" - "--pidlimit=-1" - "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/" + {{ if .CSILogRotation }} + - "--logtostderr=false" + - "--alsologtostderr=true" + - "--log_file={{ .CsiLogDirPath }}/cephcsi/log/rbd-plugin/csi-rbdplugin.log" + {{ end }} {{ if .EnableCSIAddonsSideCar }} - "--csi-addons-endpoint=$(CSIADDONS_ENDPOINT)" {{ end }} @@ -121,6 +126,10 @@ spec: mountPath: /tmp/csi/keys - name: host-run-mount mountPath: /run/mount + {{ if .CSILogRotation }} + - mountPath: {{ .CsiLogDirPath }}/cephcsi/log/rbd-plugin + name: csi-log + {{ end }} {{ if .EnablePluginSelinuxHostMount }} - name: etc-selinux mountPath: /etc/selinux @@ -154,6 +163,11 @@ spec: - "--namespace=$(POD_NAMESPACE)" - "--pod-uid=$(POD_UID)" - "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/" + {{ if .CSILogRotation }} + - "--logtostderr=false" + - "--alsologtostderr=true" + - "--log_file={{ .CsiLogDirPath }}/cephcsi/log/rbd-plugin/csi-addons.log" + {{ end }} ports: - containerPort: {{ .CSIAddonsPort }} env: @@ -179,6 +193,10 @@ spec: volumeMounts: - name: plugin-dir mountPath: /csi + {{ if .CSILogRotation }} + - mountPath: {{ .CsiLogDirPath }}/cephcsi/log/rbd-plugin + name: csi-log + {{ end }} {{ end }} {{ if .EnableLiveness }} - name: liveness-prometheus @@ -211,6 +229,13 @@ spec: hostPath: path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com" type: DirectoryOrCreate + - name: csi-log + hostPath: + path: {{ .CsiLogDirPath }}/cephcsi/log/rbd-plugin + type: DirectoryOrCreate + - name: csi-logs-logrotate + emptyDir: + type: DirectoryOrCreate - name: plugin-mount-dir hostPath: path: "{{ .KubeletDirPath }}/plugins" diff --git a/pkg/operator/ceph/csi/util.go b/pkg/operator/ceph/csi/util.go index 0eddca95ac82..f2b4c379c756 100644 --- a/pkg/operator/ceph/csi/util.go +++ b/pkg/operator/ceph/csi/util.go @@ -84,6 +84,20 @@ func templateToDeployment(name, templateData string, p templateParam) (*apps.Dep return &dep, nil } +func applyLogrotateSidecar(specTemplate *corev1.PodTemplateSpec, name, templateData string, p templateParam) { + var logrotateSidecarContainer corev1.Container + t, err := loadTemplate(name, templateData, p) + if err != nil { + panic(errors.Wrap(err, "failed to load logrotate container template")) + } + + err = yaml.Unmarshal(t, &logrotateSidecarContainer) + if err != nil { + panic(errors.Wrap(err, "failed to unmarshal logrotate container template")) + } + specTemplate.Spec.Containers = append(specTemplate.Spec.Containers, logrotateSidecarContainer) +} + func applyResourcesToContainers(opConfig map[string]string, key string, podspec *corev1.PodSpec) { resource := getComputeResource(opConfig, key) diff --git a/pkg/operator/ceph/csi/util_test.go b/pkg/operator/ceph/csi/util_test.go index 190524684bec..7ea2f09b73bc 100644 --- a/pkg/operator/ceph/csi/util_test.go +++ b/pkg/operator/ceph/csi/util_test.go @@ -120,8 +120,8 @@ func Test_applyVolumeToPodSpec(t *testing.T) { Param: CSIParam, Namespace: "foo", } - // rbdplugin has 11 volumes by default - defaultVolumes := 11 + // rbdplugin has 13 volumes by default + defaultVolumes := 13 ds, err := templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec)