diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go
index ad84d486f1..d4d05fef9a 100644
--- a/pkg/ingester/ingester_test.go
+++ b/pkg/ingester/ingester_test.go
@@ -498,6 +498,13 @@ func TestIngesterPerLabelsetLimitExceeded(t *testing.T) {
 				cortex_ingester_usage_per_labelset{labelset="{label2=\"value2\"}",limit="max_series",user="1"} 2
 				cortex_ingester_usage_per_labelset{labelset="{}",limit="max_series",user="1"} 10
 	`), "cortex_ingester_usage_per_labelset", "cortex_ingester_limits_per_labelset"))
+
+	// Force set tenant to be deleted.
+	ing.getTSDB(userID).deletionMarkFound.Store(true)
+	require.Equal(t, tsdbTenantMarkedForDeletion, ing.closeAndDeleteUserTSDBIfIdle(userID))
+	// LabelSet metrics cleaned up.
+	require.NoError(t, testutil.GatherAndCompare(registry, bytes.NewBufferString(``), "cortex_ingester_usage_per_labelset", "cortex_ingester_limits_per_labelset"))
+
 	services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck
 
 }
diff --git a/pkg/ingester/metrics.go b/pkg/ingester/metrics.go
index 3fef2c4a87..6cfe49dc1a 100644
--- a/pkg/ingester/metrics.go
+++ b/pkg/ingester/metrics.go
@@ -278,6 +278,8 @@ func (m *ingesterMetrics) deletePerUserMetrics(userID string) {
 	m.memMetadataCreatedTotal.DeleteLabelValues(userID)
 	m.memMetadataRemovedTotal.DeleteLabelValues(userID)
 	m.activeSeriesPerUser.DeleteLabelValues(userID)
+	m.usagePerLabelSet.DeletePartialMatch(prometheus.Labels{"user": userID})
+	m.limitsPerLabelSet.DeletePartialMatch(prometheus.Labels{"user": userID})
 
 	if m.memSeriesCreatedTotal != nil {
 		m.memSeriesCreatedTotal.DeleteLabelValues(userID)