Skip to content

Commit

Permalink
Merge pull request rook#14305 from parth-gr/csi-logrotate
Browse files Browse the repository at this point in the history
csi: add log rotation for csi rbd pod containers
  • Loading branch information
BlaineEXE authored Jun 27, 2024
2 parents 22125ba + d374211 commit d7eddfe
Show file tree
Hide file tree
Showing 8 changed files with 165 additions and 19 deletions.
28 changes: 17 additions & 11 deletions pkg/operator/ceph/controller/spec.go
Original file line number Diff line number Diff line change
Expand Up @@ -757,9 +757,7 @@ func PrivilegedContext(runAsRoot bool) *v1.SecurityContext {
return sec
}

// LogCollectorContainer rotate logs
func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec) *v1.Container {

func GetLogRotateConfig(c cephv1.ClusterSpec) (resource.Quantity, string) {
var maxLogSize resource.Quantity
if c.LogCollector.MaxLogSize != nil {
size := c.LogCollector.MaxLogSize.Value() / 1000 / 1000
Expand All @@ -771,20 +769,28 @@ func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec) *v1.Contai
maxLogSize = resource.MustParse(fmt.Sprintf("%dM", size))
}

rotation := "7"
if strings.Contains(daemonID, "-client.rbd-mirror") {
rotation = "28"
}

var periodicity string
if c.LogCollector.Periodicity == "1h" || c.LogCollector.Periodicity == "hourly" {
switch c.LogCollector.Periodicity {
case "1h", "hourly":
periodicity = "hourly"
} else if c.LogCollector.Periodicity == "weekly" || c.LogCollector.Periodicity == "monthly" {
case "weekly", "monthly":
periodicity = c.LogCollector.Periodicity
} else {
default:
periodicity = "daily"
}

return maxLogSize, periodicity
}

// LogCollectorContainer rotate logs
func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec) *v1.Container {
maxLogSize, periodicity := GetLogRotateConfig(c)

rotation := "7"
if strings.Contains(daemonID, "-client.rbd-mirror") {
rotation = "28"
}

logger.Debugf("setting periodicity to %q. Supported periodicity are hourly, daily, weekly and monthly", periodicity)

return &v1.Container{
Expand Down
27 changes: 26 additions & 1 deletion pkg/operator/ceph/csi/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to list ceph clusters")
}

// // Do not nothing if no ceph cluster is present
// Do nothing if no ceph cluster is present
if len(cephClusters.Items) == 0 {
logger.Debug("no ceph cluster found not deploying ceph csi driver")
EnableRBD, EnableCephFS, EnableNFS = false, false, false
Expand All @@ -217,6 +217,10 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
return reconcile.Result{}, nil
}

// if at least one cephcluster is present update the csi lograte sidecar
// with the first listed ceph cluster specs with logrotate enabled
setCSILogrotateParams(cephClusters.Items)

err = peermap.CreateOrUpdateConfig(r.opManagerContext, r.context, &peermap.PeerIDMappings{})
if err != nil {
return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to create pool ID mapping config map")
Expand Down Expand Up @@ -299,3 +303,24 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e

return reconcileResult, nil
}

func setCSILogrotateParams(cephClustersItems []cephv1.CephCluster) {
logger.Debug("set logrotate values in csi param")
spec := cephClustersItems[0].Spec
for _, cluster := range cephClustersItems {
if cluster.Spec.LogCollector.Enabled {
spec = cluster.Spec
break
}
}
CSIParam.CsiLogDirPath = spec.DataDirHostPath
if spec.DataDirHostPath == "" {
CSIParam.CsiLogDirPath = k8sutil.DataDir
}
CSIParam.CSILogRotation = spec.LogCollector.Enabled
if spec.LogCollector.Enabled {
maxSize, period := opcontroller.GetLogRotateConfig(spec)
CSIParam.CSILogRotationMaxSize = maxSize.String()
CSIParam.CSILogRotationPeriod = period
}
}
16 changes: 16 additions & 0 deletions pkg/operator/ceph/csi/spec.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ type Param struct {
ResizerImage string
DriverNamePrefix string
KubeletDirPath string
CsiLogDirPath string
ForceCephFSKernelClient string
CephFSKernelMountOptions string
CephFSPluginUpdateStrategy string
Expand Down Expand Up @@ -97,6 +98,10 @@ type Param struct {
CSICephFSPodLabels map[string]string
CSINFSPodLabels map[string]string
CSIRBDPodLabels map[string]string
CSILogRotation bool
CSILogRotationMaxSize string
CSILogRotationPeriod string
CSILogFolder string
}

type templateParam struct {
Expand Down Expand Up @@ -178,6 +183,9 @@ var (
//go:embed template/nfs/csi-nfsplugin-holder.yaml
NFSPluginHolderTemplatePath string

//go:embed template/csi-logrotate-sidecar.yaml
LogrotateTemplatePath string

holderEnabled bool
)

Expand Down Expand Up @@ -349,11 +357,19 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
if err != nil {
return errors.Wrap(err, "failed to load rbdplugin template")
}
if tp.CSILogRotation {
tp.CSILogFolder = "rbd-plugin"
applyLogrotateSidecar(&rbdPlugin.Spec.Template, "csi-rbd-daemonset-log-collector", LogrotateTemplatePath, tp)
}

rbdProvisionerDeployment, err = templateToDeployment("rbd-provisioner", RBDProvisionerDepTemplatePath, tp)
if err != nil {
return errors.Wrap(err, "failed to load rbd provisioner deployment template")
}
if tp.CSILogRotation {
tp.CSILogFolder = "rbd-provisioner"
applyLogrotateSidecar(&rbdProvisionerDeployment.Spec.Template, "csi-rbd-deployment-log-collector", LogrotateTemplatePath, tp)
}

// Create service if either liveness or GRPC metrics are enabled.
if CSIParam.EnableLiveness {
Expand Down
35 changes: 35 additions & 0 deletions pkg/operator/ceph/csi/template/csi-logrotate-sidecar.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
args:
- |
echo "Starting the csi-logrotate-sidecar"
mkdir -p {{ .CsiLogDirPath }}/cephcsi/logrotate-config/{{ .CSILogFolder }};
echo '{{ .CsiLogDirPath }}cephcsi/log/{{ .CSILogFolder }}/*.log {
{{ .CSILogRotationPeriod }}
missingok
rotate 7
compress
copytruncate
notifempty
}' > {{ .CsiLogDirPath }}/cephcsi/logrotate-config/{{ .CSILogFolder }}/csi;
echo "File creation container completed";
LOG_ROTATE_CEPH_CSI_FILE={{ .CsiLogDirPath }}/cephcsi/logrotate-config/{{ .CSILogFolder }}/csi
LOG_MAX_SIZE={{ .CSILogRotationMaxSize }}
if [ "$LOG_MAX_SIZE" != "0" ]; then
sed --in-place "4i \ \ \ \ maxsize $LOG_MAX_SIZE" "$LOG_ROTATE_CEPH_CSI_FILE"
fi
while true; do
logrotate --verbose "$LOG_ROTATE_CEPH_CSI_FILE"
sleep 15m
done
command:
- /bin/sh
- -c
image: {{ .CSIPluginImage }}
imagePullPolicy: IfNotPresent
name: log-collector
volumeMounts:
- mountPath: {{ .CsiLogDirPath }}/cephcsi/logrotate-config/{{ .CSILogFolder }}
name: csi-logs-logrotate
- mountPath: {{ .CsiLogDirPath }}/cephcsi/log/{{ .CSILogFolder }}
name: csi-log
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,9 @@ spec:
- "--extra-create-metadata=true"
- "--prevent-volume-mode-conversion=true"
- "--feature-gates=HonorPVReclaimPolicy=true"
{{- if .EnableCSITopology }}
{{ if .EnableCSITopology }}
- "--feature-gates=Topology=true"
{{- end }}
{{ end }}
env:
- name: ADDRESS
value: unix:///csi/csi-provisioner.sock
Expand Down Expand Up @@ -116,7 +116,7 @@ spec:
{{ if .EnableOMAPGenerator }}
- name: csi-omap-generator
image: {{ .CSIPluginImage }}
args :
args:
- "--type=controller"
- "--drivernamespace=$(DRIVER_NAMESPACE)"
- "--v={{ .LogLevel }}"
Expand All @@ -142,7 +142,7 @@ spec:
{{ if .EnableCSIAddonsSideCar }}
- name: csi-addons
image: {{ .CSIAddonsImage }}
args :
args:
- "--node-id=$(NODE_ID)"
- "--v={{ .LogLevel }}"
- "--csi-addons-address=$(CSIADDONS_ENDPOINT)"
Expand All @@ -155,6 +155,11 @@ spec:
- "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}"
- "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}"
- "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}"
{{ if .CSILogRotation }}
- "--logtostderr=false"
- "--alsologtostderr=true"
- "--log_file={{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner/csi-addons.log"
{{ end }}
ports:
- containerPort: {{ .CSIAddonsPort }}
env:
Expand All @@ -180,17 +185,26 @@ spec:
volumeMounts:
- name: socket-dir
mountPath: /csi
{{ if .CSILogRotation }}
- mountPath: {{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner
name: csi-log
{{ end }}
{{ end }}
- name: csi-rbdplugin
image: {{ .CSIPluginImage }}
args :
args:
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
- "--v={{ .LogLevel }}"
- "--type=rbd"
- "--controllerserver=true"
- "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com"
- "--pidlimit=-1"
{{ if .CSILogRotation }}
- "--logtostderr=false"
- "--alsologtostderr=true"
- "--log_file={{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner/csi-rbdplugin.log"
{{ end }}
{{ if .EnableCSIAddonsSideCar }}
- "--csi-addons-endpoint=$(CSIADDONS_ENDPOINT)"
{{ end }}
Expand Down Expand Up @@ -225,6 +239,10 @@ spec:
mountPath: /csi
- mountPath: /dev
name: host-dev
{{ if .CSILogRotation }}
- mountPath: {{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner
name: csi-log
{{ end }}
- mountPath: /sys
name: host-sys
- mountPath: /lib/modules
Expand Down Expand Up @@ -272,6 +290,13 @@ spec:
- name: host-dev
hostPath:
path: /dev
- name: csi-log
hostPath:
path: {{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner
type: DirectoryOrCreate
- name: csi-logs-logrotate
emptyDir:
type: DirectoryOrCreate
- name: host-sys
hostPath:
path: /sys
Expand Down
25 changes: 25 additions & 0 deletions pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,11 @@ spec:
- "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com"
- "--pidlimit=-1"
- "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/"
{{ if .CSILogRotation }}
- "--logtostderr=false"
- "--alsologtostderr=true"
- "--log_file={{ .CsiLogDirPath }}/cephcsi/log/rbd-plugin/csi-rbdplugin.log"
{{ end }}
{{ if .EnableCSIAddonsSideCar }}
- "--csi-addons-endpoint=$(CSIADDONS_ENDPOINT)"
{{ end }}
Expand Down Expand Up @@ -121,6 +126,10 @@ spec:
mountPath: /tmp/csi/keys
- name: host-run-mount
mountPath: /run/mount
{{ if .CSILogRotation }}
- mountPath: {{ .CsiLogDirPath }}/cephcsi/log/rbd-plugin
name: csi-log
{{ end }}
{{ if .EnablePluginSelinuxHostMount }}
- name: etc-selinux
mountPath: /etc/selinux
Expand Down Expand Up @@ -154,6 +163,11 @@ spec:
- "--namespace=$(POD_NAMESPACE)"
- "--pod-uid=$(POD_UID)"
- "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/"
{{ if .CSILogRotation }}
- "--logtostderr=false"
- "--alsologtostderr=true"
- "--log_file={{ .CsiLogDirPath }}/cephcsi/log/rbd-plugin/csi-addons.log"
{{ end }}
ports:
- containerPort: {{ .CSIAddonsPort }}
env:
Expand All @@ -179,6 +193,10 @@ spec:
volumeMounts:
- name: plugin-dir
mountPath: /csi
{{ if .CSILogRotation }}
- mountPath: {{ .CsiLogDirPath }}/cephcsi/log/rbd-plugin
name: csi-log
{{ end }}
{{ end }}
{{ if .EnableLiveness }}
- name: liveness-prometheus
Expand Down Expand Up @@ -211,6 +229,13 @@ spec:
hostPath:
path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com"
type: DirectoryOrCreate
- name: csi-log
hostPath:
path: {{ .CsiLogDirPath }}/cephcsi/log/rbd-plugin
type: DirectoryOrCreate
- name: csi-logs-logrotate
emptyDir:
type: DirectoryOrCreate
- name: plugin-mount-dir
hostPath:
path: "{{ .KubeletDirPath }}/plugins"
Expand Down
14 changes: 14 additions & 0 deletions pkg/operator/ceph/csi/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,20 @@ func templateToDeployment(name, templateData string, p templateParam) (*apps.Dep
return &dep, nil
}

func applyLogrotateSidecar(specTemplate *corev1.PodTemplateSpec, name, templateData string, p templateParam) {
var logrotateSidecarContainer corev1.Container
t, err := loadTemplate(name, templateData, p)
if err != nil {
panic(errors.Wrap(err, "failed to load logrotate container template"))
}

err = yaml.Unmarshal(t, &logrotateSidecarContainer)
if err != nil {
panic(errors.Wrap(err, "failed to unmarshal logrotate container template"))
}
specTemplate.Spec.Containers = append(specTemplate.Spec.Containers, logrotateSidecarContainer)
}

func applyResourcesToContainers(opConfig map[string]string, key string, podspec *corev1.PodSpec) {
resource := getComputeResource(opConfig, key)

Expand Down
4 changes: 2 additions & 2 deletions pkg/operator/ceph/csi/util_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,8 @@ func Test_applyVolumeToPodSpec(t *testing.T) {
Param: CSIParam,
Namespace: "foo",
}
// rbdplugin has 11 volumes by default
defaultVolumes := 11
// rbdplugin has 13 volumes by default
defaultVolumes := 13
ds, err := templateToDaemonSet(dsName, RBDPluginTemplatePath, tp)
assert.Nil(t, err)
applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec)
Expand Down

0 comments on commit d7eddfe

Please sign in to comment.