Make window alpha chan opaque on Wayland, fix #426

For some reason Wayland thought it would be clever to be the only
windowing system that (non-optionally) uses the alpha chan of the
window's default OpenGL framebuffer for window transparency.
This always caused glitches with dhewm3, as Doom3 uses that alpha-chan
for blending tricks (with GL_DST_ALPHA) - especially visible in the main
menu or when the flashlight is on.
So far the workaround has been r_waylandcompat which requests an OpenGL
context/visual without alpha chan (0 alpha bits), but that also causes
glitches.
There's an EGL extension that's supposed to fix this issue
(EGL_EXT_present_opaque), and newer SDL2 versions use it (when using
the wayland backend) - but unfortunately the Mesa implementation is
broken (seems to provide a visual without alpha channel even if one was
requested), see https://gitlab.freedesktop.org/mesa/mesa/-/issues/5886
and libsdl-org/SDL#4306 (comment)
for the corresponding SDL2 discussion

To work around this issue, dhewm3 now disables the use of that EGL
extension and (optionally) makes sure the alpha channel is opaque at
the end of the frame.
This behavior is controlled with the r_fillWindowAlphaChan CVar:
If it's 1, this always is done (regardless if wayland is used or not),
if it's 0 it's not done (even on wayland),
if it's -1 (the default) it's only done if the SDL "video driver" is
  wayland (this could be easily enhanced later in case other windowing
  systems have the same issue)

r_waylandcompat has been removed (it never worked properly anyway),
so now the window always has an alpha chan

neo/renderer/RenderSystem.h

@@ -88,6 +88,10 @@ typedef struct glconfig_s {
 	bool				allowARB2Path;
 
 	bool				isInitialized;
+
+	// DG: current video backend is known to need opaque default framebuffer
+	//     used if r_fillWindowAlphaChan == -1
+	bool				shouldFillWindowAlpha;
 } glconfig_t;
 
 

neo/renderer/qgl_proc.h

@@ -38,6 +38,7 @@ QGLPROC(glBegin, void, (GLenum mode))
 QGLPROC(glBindTexture, void, (GLenum target, GLuint texture))
 QGLPROC(glBitmap, void, (GLsizei width, GLsizei height, GLfloat xorig, GLfloat yorig, GLfloat xmove, GLfloat ymove, const GLubyte *bitmap))
 QGLPROC(glBlendFunc, void, (GLenum sfactor, GLenum dfactor))
+QGLPROC(glBlendEquation, void, (GLenum mode))
 QGLPROC(glCallList, void, (GLuint list))
 QGLPROC(glCallLists, void, (GLsizei n, GLenum type, const GLvoid *lists))
 QGLPROC(glClear, void, (GLbitfield mask))

neo/renderer/tr_backend.cpp

@@ -29,6 +29,8 @@ If you have questions concerning this license or the applicable additional terms
 
 #include "renderer/tr_local.h"
 
+static idCVar r_fillWindowAlphaChan( "r_fillWindowAlphaChan", "-1", CVAR_SYSTEM | CVAR_NOCHEAT | CVAR_ARCHIVE, "Make sure alpha channel of windows default framebuffer is completely opaque at the end of each frame. Needed at least when using Wayland.\n 1: do this, 0: don't do it, -1: let dhewm3 decide (default)" );
+
 frameData_t		*frameData;
 backEndState_t	backEnd;
 
@@ -529,6 +531,72 @@ const void	RB_SwapBuffers( const void *data ) {
 		RB_ShowImages();
 	}
 
+	int fillAlpha = r_fillWindowAlphaChan.GetInteger();
+	if ( fillAlpha == 1 || (fillAlpha == -1 && glConfig.shouldFillWindowAlpha) )
+	{
+		// make sure the whole alpha chan of the (default) framebuffer is opaque.
+		// at least Wayland needs this, see also the big comment in GLimp_Init()
+
+		bool blendEnabled = qglIsEnabled( GL_BLEND );
+		if ( !blendEnabled )
+			qglEnable( GL_BLEND );
+
+		// TODO: GL_DEPTH_TEST ? (should be disabled, if it needs changing at all)
+
+		bool scissorEnabled = qglIsEnabled( GL_SCISSOR_TEST );
+		if( scissorEnabled )
+			qglDisable( GL_SCISSOR_TEST );
+
+		bool tex2Denabled = qglIsEnabled( GL_TEXTURE_2D );
+		if( tex2Denabled )
+			qglDisable( GL_TEXTURE_2D );
+
+		qglDisable( GL_VERTEX_PROGRAM_ARB );
+		qglDisable( GL_FRAGMENT_PROGRAM_ARB );
+
+		qglBlendEquation( GL_FUNC_ADD );
+
+		qglBlendFunc( GL_ONE, GL_ONE );
+
+		// setup transform matrices so we can easily/reliably draw a fullscreen quad
+		qglMatrixMode( GL_MODELVIEW );
+		qglPushMatrix();
+		qglLoadIdentity();
+
+		qglMatrixMode( GL_PROJECTION );
+		qglPushMatrix();
+		qglLoadIdentity();
+		qglOrtho( 0, 1, 0, 1, -1, 1 );
+
+		// draw screen-sized quad with color (0.0, 0.0, 0.0, 1.0)
+		const float x=0, y=0, w=1, h=1;
+		qglColor4f( 0.0f, 0.0f, 0.0f, 1.0f );
+		// debug values:
+		//const float x = 0.1, y = 0.1, w = 0.8, h = 0.8;
+		//qglColor4f( 0.0f, 0.0f, 0.5f, 1.0f );
+
+		qglBegin( GL_QUADS );
+			qglVertex2f( x,   y   ); // ( 0,0 );
+			qglVertex2f( x,   y+h ); // ( 0,1 );
+			qglVertex2f( x+w, y+h ); // ( 1,1 );
+			qglVertex2f( x+w, y   ); // ( 1,0 );
+		qglEnd();
+
+		// restore previous transform matrix states
+		qglPopMatrix(); // for projection
+		qglMatrixMode( GL_MODELVIEW );
+		qglPopMatrix(); // for modelview
+
+		// restore default or previous states
+		qglBlendEquation( GL_FUNC_ADD );
+		if ( !blendEnabled )
+			qglDisable( GL_BLEND );
+		if( tex2Denabled )
+			qglEnable( GL_TEXTURE_2D );
+		if( scissorEnabled )
+			qglEnable( GL_SCISSOR_TEST );
+	}
+
 	// force a gl sync if requested
 	if ( r_finish.GetBool() ) {
 		qglFinish();

neo/sys/glimp.cpp

@@ -97,7 +97,6 @@ If you have questions concerning this license or the applicable additional terms
 
 #endif // _WIN32 and ID_ALLOW_TOOLS
 
-idCVar r_waylandcompat("r_waylandcompat", "0", CVAR_SYSTEM | CVAR_NOCHEAT | CVAR_ARCHIVE, "wayland compatible framebuffer");
 
 #if SDL_VERSION_ATLEAST(2, 0, 0)
 static SDL_Window *window = NULL;
@@ -163,6 +162,30 @@ bool GLimp_Init(glimpParms_t parms) {
 			flags |= SDL_WINDOW_FULLSCREEN;
 	}
 
+#if SDL_VERSION_ATLEAST(2, 0, 0)
+	/* Doom3 has the nasty habit of modifying the default framebuffer's alpha channel and then
+	 * relying on those modifications in blending operations (using GL_DST_(ONE_MINUS_)ALPHA).
+	 * So far that hasn't been much of a problem, because Windows, macOS, X11 etc
+	 * just ignore the alpha chan (unless maybe you explicitly tell a window it should be transparent).
+	 * Unfortunately, Wayland by default *does* use the alpha channel, which often leads to
+	 * rendering bugs (the window is partly transparent or very white in areas with low alpha).
+	 * Mesa introduced an EGL extension that's supposed to fix that (EGL_EXT_present_opaque)
+	 * and newer SDL2 versions use it by default (in the Wayland backend).
+	 * Unfortunately, the implementation of that extension is (currently?) broken (at least
+	 * in Mesa), seems like they just give you a visual without any alpha chan - which doesn't
+	 * work for Doom3, as it needs a functioning alpha chan for blending operations, see above.
+	 * See also: https://gitlab.freedesktop.org/mesa/mesa/-/issues/5886
+	 *
+	 * So to make sure dhewm3 (finally) works as expected on Wayland, we tell SDL2 to
+	 * allow transparency and then fill the alpha-chan ourselves in RB_SwapBuffers()
+	 * (unless the user disables that with r_fillWindowAlphaChan 0) */
+  #ifdef SDL_HINT_VIDEO_EGL_ALLOW_TRANSPARENCY
+	SDL_SetHint(SDL_HINT_VIDEO_EGL_ALLOW_TRANSPARENCY, "1");
+  #else // little hack so this works if the SDL2 version used for building is older than runtime version
+	SDL_SetHint("SDL_VIDEO_EGL_ALLOW_TRANSPARENCY", "1");
+  #endif
+#endif
+
 	int colorbits = 24;
 	int depthbits = 24;
 	int stencilbits = 8;
@@ -227,7 +250,7 @@ bool GLimp_Init(glimpParms_t parms) {
 		if (tcolorbits == 24)
 			channelcolorbits = 8;
 
-		int talphabits = r_waylandcompat.GetBool() ? 0 : channelcolorbits;
+		int talphabits = channelcolorbits;
 
 try_again:
 
@@ -535,6 +558,15 @@ bool GLimp_Init(glimpParms_t parms) {
 
 		glConfig.displayFrequency = 0;
 
+		// for r_fillWindowAlphaChan -1, see also the big comment above
+		glConfig.shouldFillWindowAlpha = false;
+#if SDL_VERSION_ATLEAST(2, 0, 0)
+		const char* videoDriver = SDL_GetCurrentVideoDriver();
+		if (idStr::Icmp(videoDriver, "wayland") == 0) {
+			glConfig.shouldFillWindowAlpha = true;
+		}
+#endif
+
 		break;
 	}
 

neo/sys/stub/stub_gl.cpp

@@ -44,6 +44,7 @@ void APIENTRY glBegin(GLenum mode){};
 void APIENTRY glBindTexture(GLenum target, GLuint texture){};
 void APIENTRY glBitmap(GLsizei width, GLsizei height, GLfloat xorig, GLfloat yorig, GLfloat xmove, GLfloat ymove, const GLubyte *bitmap){};
 void APIENTRY glBlendFunc(GLenum sfactor, GLenum dfactor){};
+void APIENTRY glBlendEquation(GLenum mode){};
 void APIENTRY glCallList(GLuint list){};
 void APIENTRY glCallLists(GLsizei n, GLenum type, const GLvoid *lists){};
 void APIENTRY glClear(GLbitfield mask){};