diff --git a/geometry/render/render_engine.h b/geometry/render/render_engine.h
index 29c725ff14d4..3248900d33a1 100644
--- a/geometry/render/render_engine.h
+++ b/geometry/render/render_engine.h
@@ -334,7 +334,12 @@ class RenderEngine : public ShapeReifier {
    are encoded with unsigned bytes in the range [0, 255] per channel or
    _double-valued_ such that each channel is encoded with a double in the range
    [0, 1]. Conversion to RenderLabel is only supported from byte-valued color
-   values.  */
+   values.
+
+   These utilities are provided as a _convenience_ to derived classes. Derived
+   classes are not required to encode labels as colors in the same way. They are
+   only obliged to return label images with proper label values according to
+   the documented semantics.  */
   //@{
 
   /** Transforms the given byte-valued RGB color value into its corresponding
diff --git a/geometry/render_gltf_client/BUILD.bazel b/geometry/render_gltf_client/BUILD.bazel
index 21e422a024f4..6e3054a52a4a 100644
--- a/geometry/render_gltf_client/BUILD.bazel
+++ b/geometry/render_gltf_client/BUILD.bazel
@@ -188,6 +188,7 @@ drake_cc_library(
     srcs = ["test/internal_sample_image_data.cc"],
     hdrs = ["test/internal_sample_image_data.h"],
     deps = [
+        "//geometry/render:render_label",
         "//systems/sensors:image",
     ],
 )
@@ -235,7 +236,6 @@ drake_cc_binary(
     testonly = True,
     srcs = ["test/server_vtk_backend.cc"],
     deps = [
-        "//geometry/render:render_label",
         "//geometry/render/shaders:depth_shaders",
         "//geometry/render_vtk:factory",
         "//geometry/render_vtk:internal_render_engine_vtk",
diff --git a/geometry/render_gltf_client/internal_render_engine_gltf_client.cc b/geometry/render_gltf_client/internal_render_engine_gltf_client.cc
index 2d00e9f02b75..55590c44a8a2 100644
--- a/geometry/render_gltf_client/internal_render_engine_gltf_client.cc
+++ b/geometry/render_gltf_client/internal_render_engine_gltf_client.cc
@@ -347,14 +347,19 @@ void RenderEngineGltfClient::DoRenderLabelImage(
   ImageRgba8U colored_label_image(width, height);
   render_client_->LoadColorImage(image_path, &colored_label_image);
 
+  // By convention (see render_gltf_client_doxygen.h), server-only artifacts are
+  // colored white to indicate the "don't care" semantic.
   // Convert from RGB to Label.
+  const ColorI kDontCare{255, 255, 255};
   ColorI color;
   for (int y = 0; y < height; ++y) {
     for (int x = 0; x < width; ++x) {
       color.r = colored_label_image.at(x, y)[0];
       color.g = colored_label_image.at(x, y)[1];
       color.b = colored_label_image.at(x, y)[2];
-      label_image_out->at(x, y)[0] = RenderEngine::LabelFromColor(color);
+      label_image_out->at(x, y)[0] = color == kDontCare
+                                         ? render::RenderLabel::kDontCare
+                                         : RenderEngine::LabelFromColor(color);
     }
   }
 
diff --git a/geometry/render_gltf_client/render_gltf_client_doxygen.h b/geometry/render_gltf_client/render_gltf_client_doxygen.h
index 6a19eeaef5af..b21a9de57254 100644
--- a/geometry/render_gltf_client/render_gltf_client_doxygen.h
+++ b/geometry/render_gltf_client/render_gltf_client_doxygen.h
@@ -50,6 +50,8 @@ its own.
     - [Allowed Image Response Types](#allowed-image-response-types)
     - [Notes on glTF Camera Specification](#notes-on-gltf-camera-specification)
     - [Notes on Communicating Errors](#notes-on-communicating-errors)
+- [Notes on Rendering Label Images](#notes-on-rendering-label-images)
+- [Existing Server Implementations](#existing-server-implementations)
 - [Developing your own Server](#developing-your-own-server)
 
 <h2 id="server-api">Server API</h2>
@@ -253,12 +255,12 @@ The client accepts the following image types from a server render:
       interpret this rendering as units of millimeters.  Pixels at their maximum
       value (2¹⁶-1) will be interpreted as kTooFar (i.e., infinity).
 - When `image_type="label"`, the server may return:
-    - An RGB (3 channel) unsigned char PNG image.  The client will interpret
-      this rendering as a colored label image and convert to the final label
-      image.
-    - An RGBA (4 channel) unsigned char PNG image.  The client will interpret
-      this rendering as a colored label image and convert to the final label
-      image.
+    - An RGB (3 channel) unsigned char PNG image.
+    - An RGBA (4 channel) unsigned char PNG image.
+    - Note: The client will interpret this rendering as a colored label image
+    and convert to the final label image. See
+    [this section](#notes-on-rendering-label-images) for other requirements for
+    returning a label image.
 
 <h3 id="notes-on-gltf-camera-specification">Notes on glTF Camera Specification</h3>
 <hr>
@@ -353,6 +355,44 @@ required, the user of the server will have no hints as to what is going wrong
 with the client-server communication.  When the file response is provided, this
 information will be included in the exception message produced by the client.
 
+<h2 id="notes-on-rendering-label-images">Notes on Rendering Label Images</h2>
+<hr>
+
+Renderers typically can't render objects with "labels". Drake encodes the labels
+associated with geometries as unique colors and provides those colors to the
+server as attributes on the meshes. Thus, the label output from any server will
+be an RGB or RGBA PNG.
+
+__All renderable artifacts that exist _only_ in the server -- that are not part
+of the Drake-provided glTF -- must be colored white (RGB=(255, 255, 255))__.
+These server-only renderable artifacts include:
+
+  - The background color.
+  - Any geometries that the server loads (e.g., walls in a room, environment
+    images, etc.).
+
+When producing the final label output, the client will interpret this particular
+RGB value as render::RenderLabel::kDontCare. This means that a remote server
+will never report a pixel with the render::RenderLabel::kEmpty value.
+
+For an image to be a proper color-encoded label image, the only pixel values in
+the image must be one of the recognized label encodings. This may require
+special render configurations. Any configurations that can introduce color
+variation must be disabled. That includes (but is not limited to) the following
+render features:
+
+  - Anti-aliasing (multi-sampling)
+  - Lighting
+  - Color transformations on the rendered image
+
+<h2 id="existing-server-implementations">Existing Server Implementations</h2>
+<hr>
+
+[drake-blender] is a glTF render server using [Blender] as the backend.
+
+[drake-blender]: https://github.com/RobotLocomotion/drake-blender
+[Blender]: https://www.blender.org
+
 <h2 id="developing-your-own-server">Developing your own Server</h2>
 <hr>
 
diff --git a/geometry/render_gltf_client/test/integration_test.py b/geometry/render_gltf_client/test/integration_test.py
index c3332af5f706..17e6282e69bd 100644
--- a/geometry/render_gltf_client/test/integration_test.py
+++ b/geometry/render_gltf_client/test/integration_test.py
@@ -261,6 +261,13 @@ def test_integration(self):
             )
             self.assert_error_fraction_less(depth_diff, INVALID_PIXEL_FRACTION)
 
+            # By convention, where RenderEngineVtk uses RenderLabel::kEmpty
+            # (32766), RenderEngineGltfClient uses RenderLabel::kDontCare
+            # (32764). The values are hard-coded intentionally to avoid having
+            # the large dependency of pydrake. Keep the values in sync if we
+            # ever change the implementation of RenderLabel.
+            # Make them match to facilitate comparison.
+            vtk_label[vtk_label == 32766] = 32764
             label_diff = (
                 np.absolute(vtk_label - client_label) > LABEL_PIXEL_THRESHOLD
             )
diff --git a/geometry/render_gltf_client/test/internal_render_engine_gltf_client_test.cc b/geometry/render_gltf_client/test/internal_render_engine_gltf_client_test.cc
index 3b0fa1c777ad..b11d5920d5b2 100644
--- a/geometry/render_gltf_client/test/internal_render_engine_gltf_client_test.cc
+++ b/geometry/render_gltf_client/test/internal_render_engine_gltf_client_test.cc
@@ -281,7 +281,9 @@ TEST_F(RenderEngineGltfClientTest, DoRenderLabelImage) {
     // Make sure the temporary directory is / is not being cleaned up.
     CheckExpectedFiles(engine.temp_directory(), cleanup, ".png");
 
-    // Make sure the image got loaded as expected.
+    // Make sure the image got loaded as expected. Note that it also tests
+    // whether a server-returned white pixel is properly converted to
+    // render::RenderLabel::kDontCare.
     EXPECT_EQ(label_image, CreateTestLabelImage());
   }
 }
diff --git a/geometry/render_gltf_client/test/internal_sample_image_data.cc b/geometry/render_gltf_client/test/internal_sample_image_data.cc
index 9df5fa94b3e8..292365f8d214 100644
--- a/geometry/render_gltf_client/test/internal_sample_image_data.cc
+++ b/geometry/render_gltf_client/test/internal_sample_image_data.cc
@@ -3,6 +3,7 @@
 #include <array>
 #include <vector>
 
+#include "drake/geometry/render/render_label.h"
 #include "drake/systems/sensors/image.h"
 
 namespace drake {
@@ -59,7 +60,7 @@ ImageDepth32F CreateTestDepthImage() {
 ImageLabel16I CreateTestLabelImage() {
   ImageLabel16I test_label_image{kTestImageWidth, kTestImageHeight};
   using T = ImageLabel16I::T;
-  std::vector<T> image_data{0, 1, 2, 3, 4, 5};
+  std::vector<T> image_data{0, 1, 2, 3, 4, render::RenderLabel::kDontCare};
   int p = 0;
   for (int y = 0; y < 2; ++y) {
     for (int x = 0; x < 3; ++x) {
diff --git a/geometry/render_gltf_client/test/internal_sample_image_data.h b/geometry/render_gltf_client/test/internal_sample_image_data.h
index edefead3d35f..6e6c4d228fff 100644
--- a/geometry/render_gltf_client/test/internal_sample_image_data.h
+++ b/geometry/render_gltf_client/test/internal_sample_image_data.h
@@ -39,12 +39,14 @@ systems::sensors::ImageRgba8U CreateTestColorImage(bool pad_alpha);
 systems::sensors::ImageDepth32F CreateTestDepthImage();
 
 /* Recreates an ImageLabel16I image that should be exactly the same as the
- loaded test_colored_label_rgba_8U.png after it has been converted.
+ loaded test_colored_label_rgba_8U.png after it has been converted. Note that
+ the value of the sixth pixel is set to render::RenderLabel::kDontCare to test
+ the special conversion of white color (see render_gltf_client_doxygen.h).
 
      ┌─────────────────┬─────────────────┬─────────────────┐
      │        0        │        1        │        2        │
      ├─────────────────┼─────────────────┼─────────────────┤
-     │        3        │        4        │        5        │
+     │        3        │        4        │    kDontCare    │
      └─────────────────┴─────────────────┴─────────────────┘
 
  Returns an image (width=3, height=2) where every label value is unique. */
diff --git a/geometry/render_gltf_client/test/server_vtk_backend.cc b/geometry/render_gltf_client/test/server_vtk_backend.cc
index f02ef3be3009..2926700e3724 100644
--- a/geometry/render_gltf_client/test/server_vtk_backend.cc
+++ b/geometry/render_gltf_client/test/server_vtk_backend.cc
@@ -274,12 +274,9 @@ int DoMain() {
       }
     }
   } else {  // FLAGS_image_type == "label"
-    // TODO(zachfang): We need to find a workaround and document it, so that no
-    // server implementations need to hard-code these magic numbers.
-    /* NOTE: This is hard-coded to be the value that
-     RenderEngine::GetColorDFromLabel(RenderLabel::kEmpty) would produce.  If
-     that value changes, this should change to match. */
-    renderer->SetBackground(254.0 / 255.0, 127.0 / 255.0, 0.0);
+    // Following the client-server API (see render_gltf_client_doxygen.h), the
+    // background of a label image should be set to white.
+    renderer->SetBackground(1.0, 1.0, 1.0);
 
     // Same as RenderEngineVtk, label actors have lighting disabled.  Labels
     // have already been encoded as geometry materials in the glTF.  By
diff --git a/geometry/render_gltf_client/test/test_colored_label_rgba_8U.png b/geometry/render_gltf_client/test/test_colored_label_rgba_8U.png
index 8f2632d540c1..d87c983a8e39 100644
Binary files a/geometry/render_gltf_client/test/test_colored_label_rgba_8U.png and b/geometry/render_gltf_client/test/test_colored_label_rgba_8U.png differ