diff --git a/drivers/gpu/drm/vc4/vc4_drv.h b/drivers/gpu/drm/vc4/vc4_drv.h
index 0868cdbf1fce30..90c4bc7d31b10a 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.h
+++ b/drivers/gpu/drm/vc4/vc4_drv.h
@@ -365,6 +365,8 @@ struct vc4_hvs {
 		unsigned int enabled: 1;
 	} eof_irq[HVS_NUM_CHANNELS];
 
+	bool bg_fill[HVS_NUM_CHANNELS];
+
 	unsigned long max_core_rate;
 
 	/* Memory manager for CRTCs to allocate space in the display
diff --git a/drivers/gpu/drm/vc4/vc4_hvs.c b/drivers/gpu/drm/vc4/vc4_hvs.c
index 02a9bb43035d19..b7f41b0b696372 100644
--- a/drivers/gpu/drm/vc4/vc4_hvs.c
+++ b/drivers/gpu/drm/vc4/vc4_hvs.c
@@ -1509,14 +1509,7 @@ void vc4_hvs_atomic_flush(struct drm_crtc *crtc,
 		/* This sets a black background color fill, as is the case
 		 * with other DRM drivers.
 		 */
-		if (enable_bg_fill)
-			HVS_WRITE(SCALER6_DISPX_CTRL1(channel),
-				  HVS_READ(SCALER6_DISPX_CTRL1(channel)) |
-				  SCALER6(DISPX_CTRL1_BGENB));
-		else
-			HVS_WRITE(SCALER6_DISPX_CTRL1(channel),
-				  HVS_READ(SCALER6_DISPX_CTRL1(channel)) &
-				  ~SCALER6(DISPX_CTRL1_BGENB));
+		hvs->bg_fill[channel] = enable_bg_fill;
 	} else {
 		/* we can actually run with a lower core clock when background
 		 * fill is enabled on VC4_GEN_5 so leave it enabled always.
@@ -1700,6 +1693,15 @@ static irqreturn_t vc6_hvs_eof_irq_handler(int irq, void *data)
 		if (hvs->eof_irq[i].desc != irq)
 			continue;
 
+		if (hvs->bg_fill[i])
+			HVS_WRITE(SCALER6_DISPX_CTRL1(i),
+				  HVS_READ(SCALER6_DISPX_CTRL1(i)) |
+				  SCALER6(DISPX_CTRL1_BGENB));
+		else
+			HVS_WRITE(SCALER6_DISPX_CTRL1(i),
+				  HVS_READ(SCALER6_DISPX_CTRL1(i)) &
+				  ~SCALER6(DISPX_CTRL1_BGENB));
+
 		vc4_hvs_schedule_dlist_sweep(hvs, i);
 		return IRQ_HANDLED;
 	}