x11: Use glXChooseFBConfig when available in X11_GL_GetVisual

When choosing an X11 Visual for a window based on its GLX capabilities, first
try glXChooseFBConfig (if available) before falling back to glXChooseVisual.
This normally does not make a difference because most GLX drivers create a
Visual for every GLXFBConfig, exposing all of the same capabilities.

For GLX render offload configurations (also know as "PRIME") where one GPU is
providing GLX rendering support for windows on an X screen running on a
different GPU, the GPU doing the offloading needs to use the Visuals that were
created by the host GPU's driver rather than being able to add its own. This
means that there may be fewer Visuals available for all of the GLXFBConfigs the
guest driver wants to expose. In order to handle that situation, the NVIDIA GLX
driver creates many GLXFBConfigs that map to the same Visual when running in a
render offload configuration.

This can result in a glXChooseVisual request failing to find a supported Visual
when there is a GLXFBConfig for that configuration that would have worked. For
example, when the game "Unnamed SDVX Clone" [1] tries to create a configuration
with multisample, glXChooseVisual fails because the Visual assigned to the
multisample GLXFBConfigs is shared with the GLXFBConfigs without multisample.

Avoid this problem by using glXChooseFBConfig, when available, to find a
GLXFBConfig with the requested capabilities and then using
glXGetVisualFromFBConfig to find the corresponding X11 Visual. This allows the
game to run, although it doesn't make me any better at actually playing it...

Signed-off-by: Aaron Plattner <[email protected]>
Fixes: https://forums.developer.nvidia.com/t/prime-run-cannot-create-window-x-glxcreatecontext/180214

[1] https://github.com/Drewol/unnamed-sdvx-clone

src/video/x11/SDL_x11opengl.c

@@ -1,6 +1,7 @@
 /*
   Simple DirectMedia Layer
   Copyright (C) 1997-2021 Sam Lantinga <[email protected]>
+  Copyright (C) 2021 NVIDIA Corporation
 
   This software is provided 'as-is', without any express or implied
   warranty.  In no event will the authors be held liable for any damages
@@ -420,6 +421,9 @@ X11_GL_InitExtensions(_THIS)
         _this->gl_data->glXChooseFBConfig =
             (GLXFBConfig *(*)(Display *, int, const int *, int *))
                 X11_GL_GetProcAddress(_this, "glXChooseFBConfig");
+        _this->gl_data->glXGetVisualFromFBConfig =
+            (XVisualInfo *(*)(Display *, GLXFBConfig))
+                X11_GL_GetProcAddress(_this, "glXGetVisualFromFBConfig");
     }
 
     /* Check for GLX_EXT_visual_rating */
@@ -598,20 +602,41 @@ X11_GL_GetVisual(_THIS, Display * display, int screen)
 {
     /* 64 seems nice. */
     int attribs[64];
-    XVisualInfo *vinfo;
+    XVisualInfo *vinfo = NULL;
     int *pvistypeattr = NULL;
 
     if (!_this->gl_data) {
         /* The OpenGL library wasn't loaded, SDL_GetError() should have info */
         return NULL;
     }
 
-    X11_GL_GetAttributes(_this, display, screen, attribs, 64, SDL_FALSE, &pvistypeattr);
-    vinfo = _this->gl_data->glXChooseVisual(display, screen, attribs);
+    if (_this->gl_data->glXChooseFBConfig &&
+        _this->gl_data->glXGetVisualFromFBConfig) {
+        GLXFBConfig *framebuffer_config = NULL;
+        int fbcount = 0;
+
+        X11_GL_GetAttributes(_this, display, screen, attribs, 64, SDL_TRUE, &pvistypeattr);
+        framebuffer_config = _this->gl_data->glXChooseFBConfig(display, screen, attribs, &fbcount);
+        if (!framebuffer_config && (pvistypeattr != NULL)) {
+            *pvistypeattr = None;
+            framebuffer_config = _this->gl_data->glXChooseFBConfig(display, screen, attribs, &fbcount);
+        }
+
+        if (framebuffer_config) {
+            vinfo = _this->gl_data->glXGetVisualFromFBConfig(display, framebuffer_config[0]);
+        }
+
+        X11_XFree(framebuffer_config);
+    }
 
-    if (!vinfo && (pvistypeattr != NULL)) {
-        *pvistypeattr = None;
+    if (!vinfo) {
+        X11_GL_GetAttributes(_this, display, screen, attribs, 64, SDL_FALSE, &pvistypeattr);
         vinfo = _this->gl_data->glXChooseVisual(display, screen, attribs);
+
+        if (!vinfo && (pvistypeattr != NULL)) {
+            *pvistypeattr = None;
+            vinfo = _this->gl_data->glXChooseVisual(display, screen, attribs);
+        }
     }
 
     if (!vinfo) {

src/video/x11/SDL_x11opengl.h

@@ -53,6 +53,7 @@ struct SDL_GLDriverData
     GLXContext (*glXCreateContext) (Display*,XVisualInfo*,GLXContext,Bool);
     GLXContext (*glXCreateContextAttribsARB) (Display*,GLXFBConfig,GLXContext,Bool,const int *);
     GLXFBConfig *(*glXChooseFBConfig) (Display*,int,const int *,int *);
+    XVisualInfo *(*glXGetVisualFromFBConfig) (Display*,GLXFBConfig);
     void (*glXDestroyContext) (Display*, GLXContext);
     Bool(*glXMakeCurrent) (Display*,GLXDrawable,GLXContext);
     void (*glXSwapBuffers) (Display*, GLXDrawable);