Move screen buffer metadata to attributes buffer, output real RGBA data

pyboy/api/screen.py

@@ -61,11 +61,14 @@ def __init__(self, mb):
         Returns
         -------
         str:
-            Color format of the raw screen buffer. E.g. 'RGBX'.
+            Color format of the raw screen buffer. E.g. 'RGBA'.
         """
         self.image = None
         """
-        Generates a PIL Image from the screen buffer. The screen buffer is internally row-major, but PIL hides this.
+        Reference to a PIL Image from the screen buffer. **Remember to copy, resize or convert this object** if you
+        intend to store it. The backing buffer will update, but it will be the same `PIL.Image` object.
+
+        The screen buffer is internally row-major, but PIL hides this.
 
         Convenient for screen captures, but might be a bottleneck, if you use it to train a neural network. In which
         case, read up on the `pyboy.api` features, [Pan Docs](https://gbdev.io/pandocs/) on tiles/sprites,
@@ -90,8 +93,10 @@ def __init__(self, mb):
             dtype=np.uint8,
         ).reshape(ROWS, COLS, 4)
         """
-        Provides the screen data in NumPy format. The format is given by `pyboy.api.screen.Screen.raw_buffer_format`.
-        The screen buffer is row-major.
+        References the screen data in NumPy format. **Remember to copy this object** if you intend to store it.
+        The backing buffer will update, but it will be the same `ndarray` object.
+
+        The format is given by `pyboy.api.screen.Screen.raw_buffer_format`. The screen buffer is row-major.
 
         Returns
         -------

pyboy/api/tile.py

@@ -97,7 +97,7 @@ def __init__(self, mb, identifier):
         Returns
         -------
         str:
-            Color format of the raw screen buffer. E.g. 'RGBX'.
+            Color format of the raw screen buffer. E.g. 'RGBA'.
         """
 
     def image(self):
@@ -116,9 +116,9 @@ def image(self):
             return None
 
         if cythonmode:
-            return Image.fromarray(self._image_data().base, mode="RGBX")
+            return Image.fromarray(self._image_data().base, mode=self.raw_buffer_format)
         else:
-            return Image.frombytes("RGBX", (8, 8), self._image_data())
+            return Image.frombytes(self.raw_buffer_format, (8, 8), self._image_data())
 
     def image_ndarray(self):
         """

pyboy/core/lcd.pxd

@@ -116,9 +116,11 @@ cdef class Renderer:
     cdef bint cgb
 
     cdef array _screenbuffer_raw
+    cdef array _screenbuffer_attributes_raw
     cdef object _screenbuffer_ptr
     cdef array _tilecache0_raw, _spritecache0_raw, _spritecache1_raw
     cdef uint32_t[:,:] _screenbuffer
+    cdef uint8_t[:,:] _screenbuffer_attributes
     cdef uint32_t[:,:] _tilecache0, _spritecache0, _spritecache1
 
     cdef int[10] sprites_to_render
@@ -147,6 +149,7 @@ cdef class Renderer:
         yy=int,
         tilecache=uint32_t[:,:],
         bg_priority_apply=uint32_t,
+        col0=uint8_t,
     )
     cdef void scanline(self, LCD, int) noexcept nogil
 
@@ -172,7 +175,7 @@ cdef class Renderer:
         pixel=uint32_t,
         bgmappriority=bint,
     )
-    cdef void scanline_sprites(self, LCD, int, uint32_t[:,:], bint) noexcept nogil
+    cdef void scanline_sprites(self, LCD, int, uint32_t[:,:], uint8_t[:,:], bint) noexcept nogil
     cdef void sort_sprites(self, int) noexcept nogil
 
     cdef void clear_cache(self) noexcept nogil

pyboy/core/lcd.py

@@ -23,10 +23,11 @@
 
 
 def rgb_to_bgr(color):
+    a = 0xFF
     r = (color >> 16) & 0xFF
     g = (color >> 8) & 0xFF
     b = color & 0xFF
-    return (b << 16) | (g << 8) | r
+    return (a << 24) | (b << 16) | (g << 8) | r
 
 
 class LCD:
@@ -166,7 +167,9 @@ def tick(self, cycles):
                     self.clock_target += 206 * multiplier
 
                     self.renderer.scanline(self, self.LY)
-                    self.renderer.scanline_sprites(self, self.LY, self.renderer._screenbuffer, False)
+                    self.renderer.scanline_sprites(
+                        self, self.LY, self.renderer._screenbuffer, self.renderer._screenbuffer_attributes, False
+                    )
                     if self.LY < 143:
                         self.next_stat_mode = 2
                     else:
@@ -305,10 +308,7 @@ def get(self):
         return self.value
 
     def getcolor(self, i):
-        if i==0:
-            return self.palette_mem_rgb[self.lookup[0]] | COL0_FLAG
-        else:
-            return self.palette_mem_rgb[self.lookup[i]]
+        return self.palette_mem_rgb[self.lookup[i]]
 
 
 class STATRegister:
@@ -371,14 +371,14 @@ def _get_sprite_height(self):
         return self.sprite_height
 
 
-COL0_FLAG = 0b01 << 24
-BG_PRIORITY_FLAG = 0b10 << 24
+COL0_FLAG = 0b01
+BG_PRIORITY_FLAG = 0b10
 
 
 class Renderer:
     def __init__(self, cgb):
         self.cgb = cgb
-        self.color_format = "RGBX"
+        self.color_format = "RGBA"
 
         self.buffer_dims = (ROWS, COLS)
 
@@ -387,6 +387,7 @@ def __init__(self, cgb):
 
         # Init buffers as white
         self._screenbuffer_raw = array("B", [0x00] * (ROWS*COLS*4))
+        self._screenbuffer_attributes_raw = array("B", [0x00] * (ROWS*COLS))
         self._tilecache0_raw = array("B", [0x00] * (TILES*8*8*4))
         self._spritecache0_raw = array("B", [0x00] * (TILES*8*8*4))
         self._spritecache1_raw = array("B", [0x00] * (TILES*8*8*4))
@@ -398,6 +399,7 @@ def __init__(self, cgb):
         self.clear_cache()
 
         self._screenbuffer = memoryview(self._screenbuffer_raw).cast("I", shape=(ROWS, COLS))
+        self._screenbuffer_attributes = memoryview(self._screenbuffer_attributes_raw).cast("B", shape=(ROWS, COLS))
         self._tilecache0 = memoryview(self._tilecache0_raw).cast("I", shape=(TILES * 8, 8))
         # OBP0 palette
         self._spritecache0 = memoryview(self._spritecache0_raw).cast("I", shape=(TILES * 8, 8))
@@ -470,6 +472,7 @@ def scanline(self, lcd, y):
                     yy = (8*wt + (7 - (self.ly_window) % 8)) if vertflip else (8*wt + (self.ly_window) % 8)
 
                     pixel = lcd.bcpd.getcolor(palette, tilecache[yy, xx])
+                    col0 = (tilecache[yy, xx] == 0) & 1
                     if bg_priority:
                         # We hide extra rendering information in the lower 8 bits (A) of the 32-bit RGBA format
                         bg_priority_apply = BG_PRIORITY_FLAG
@@ -478,8 +481,14 @@ def scanline(self, lcd, y):
                     xx = (x-wx) % 8
                     yy = 8*wt + (self.ly_window) % 8
                     pixel = lcd.BGP.getcolor(self._tilecache0[yy, xx])
-
-                self._screenbuffer[y, x] = pixel | bg_priority_apply
+                    col0 = (self._tilecache0[yy, xx] == 0) & 1
+
+                self._screenbuffer[y, x] = pixel
+                # COL0_FLAG is 1
+                self._screenbuffer_attributes[y, x] = bg_priority_apply | col0
+                # self._screenbuffer_attributes[y, x] = bg_priority_apply
+                # if col0:
+                #     self._screenbuffer_attributes[y, x] = self._screenbuffer_attributes[y, x] | col0
             # background_enable doesn't exist for CGB. It works as master priority instead
             elif (not self.cgb and lcd._LCDC.background_enable) or self.cgb:
                 tile_addr = background_offset + (y+by) // 8 * 32 % 0x400 + (x+bx) // 8 % 32
@@ -506,6 +515,7 @@ def scanline(self, lcd, y):
                     yy = (8*bt + (7 - (y+by) % 8)) if vertflip else (8*bt + (y+by) % 8)
 
                     pixel = lcd.bcpd.getcolor(palette, tilecache[yy, xx])
+                    col0 = (tilecache[yy, xx] == 0) & 1
                     if bg_priority:
                         # We hide extra rendering information in the lower 8 bits (A) of the 32-bit RGBA format
                         bg_priority_apply = BG_PRIORITY_FLAG
@@ -514,11 +524,14 @@ def scanline(self, lcd, y):
                     xx = (x+offset) % 8
                     yy = 8*bt + (y+by) % 8
                     pixel = lcd.BGP.getcolor(self._tilecache0[yy, xx])
+                    col0 = (self._tilecache0[yy, xx] == 0) & 1
 
-                self._screenbuffer[y, x] = pixel | bg_priority_apply
+                self._screenbuffer[y, x] = pixel
+                self._screenbuffer_attributes[y, x] = bg_priority_apply | col0
             else:
                 # If background is disabled, it becomes white
                 self._screenbuffer[y, x] = lcd.BGP.getcolor(0)
+                self._screenbuffer_attributes[y, x] = 0
 
         if y == 143:
             # Reset at the end of a frame. We set it to -1, so it will be 0 after the first increment
@@ -541,7 +554,7 @@ def sort_sprites(self, sprite_count):
             # Insert the key into its correct position in the sorted portion
             self.sprites_to_render[j + 1] = key
 
-    def scanline_sprites(self, lcd, ly, buffer, ignore_priority):
+    def scanline_sprites(self, lcd, ly, buffer, buffer_attributes, ignore_priority):
         if not lcd._LCDC.sprite_enable or lcd.disable_renderer:
             return
 
@@ -621,14 +634,14 @@ def scanline_sprites(self, lcd, ly, buffer, ignore_priority):
                 if 0 <= x < COLS and not color_code == 0: # If pixel is not transparent
                     if self.cgb:
                         pixel = lcd.ocpd.getcolor(palette, color_code)
-                        bgmappriority = buffer[ly, x] & BG_PRIORITY_FLAG
+                        bgmappriority = buffer_attributes[ly, x] & BG_PRIORITY_FLAG
 
                         if lcd._LCDC.cgb_master_priority: # If 0, sprites are always on top, if 1 follow priorities
                             if bgmappriority: # If 0, use spritepriority, if 1 take priority
-                                if buffer[ly, x] & COL0_FLAG:
+                                if buffer_attributes[ly, x] & COL0_FLAG:
                                     buffer[ly, x] = pixel
                             elif spritepriority: # If 1, sprite is behind bg/window. Color 0 of window/bg is transparent
-                                if buffer[ly, x] & COL0_FLAG:
+                                if buffer_attributes[ly, x] & COL0_FLAG:
                                     buffer[ly, x] = pixel
                             else:
                                 buffer[ly, x] = pixel
@@ -642,7 +655,7 @@ def scanline_sprites(self, lcd, ly, buffer, ignore_priority):
                             pixel = lcd.OBP0.getcolor(color_code)
 
                         if spritepriority: # If 1, sprite is behind bg/window. Color 0 of window/bg is transparent
-                            if buffer[ly, x] & COL0_FLAG: # if BG pixel is transparent
+                            if buffer_attributes[ly, x] & COL0_FLAG: # if BG pixel is transparent
                                 buffer[ly, x] = pixel
                         else:
                             buffer[ly, x] = pixel
@@ -735,6 +748,7 @@ def blank_screen(self, lcd):
         for y in range(ROWS):
             for x in range(COLS):
                 self._screenbuffer[y, x] = lcd.BGP.getcolor(0)
+                self._screenbuffer_attributes[y, x] = 0
 
     def save_state(self, f):
         for y in range(ROWS):
@@ -748,6 +762,7 @@ def save_state(self, f):
         for y in range(ROWS):
             for x in range(COLS):
                 f.write_32bit(self._screenbuffer[y, x])
+                f.write(self._screenbuffer_attributes[y, x])
 
     def load_state(self, f, state_version):
         if state_version >= 2:
@@ -764,6 +779,8 @@ def load_state(self, f, state_version):
             for y in range(ROWS):
                 for x in range(COLS):
                     self._screenbuffer[y, x] = f.read_32bit()
+                    if state_version >= 10:
+                        self._screenbuffer_attributes[y, x] = f.read()
 
         self.clear_cache()
 
@@ -958,13 +975,12 @@ def __init__(self, i_reg):
                 self.palette_mem_rgb[n + m] = self.cgb_to_rgb(c[m], m)
 
     def cgb_to_rgb(self, cgb_color, index):
+        alpha = 0xFF
         red = (cgb_color & 0x1F) << 3
         green = ((cgb_color >> 5) & 0x1F) << 3
         blue = ((cgb_color >> 10) & 0x1F) << 3
         # NOTE: Actually BGR, not RGB
-        rgb_color = ((blue << 16) | (green << 8) | red)
-        if index % 4 == 0:
-            rgb_color |= COL0_FLAG
+        rgb_color = ((alpha << 24) | (blue << 16) | (green << 8) | red)
         return rgb_color
 
     def set(self, val):

pyboy/plugins/debug.pxd

@@ -56,8 +56,8 @@ cdef class BaseDebugWindow(PyBoyWindowPlugin):
     cdef object _window
     cdef object _sdlrenderer
     cdef object _sdltexturebuffer
-    cdef array buf
     cdef uint32_t[:,:] buf0
+    cdef uint8_t[:,:] buf0_attributes
     cdef object buf_p
 
     @cython.locals(y=int, x=int, _y=int, _x=int)

pyboy/plugins/debug.py

@@ -355,11 +355,14 @@ def handle_breakpoint(self):
                 self.mb.tick()
 
 
-def make_buffer(w, h):
-    buf = array("B", [0x55] * (w*h*4))
-    buf0 = memoryview(buf).cast("I", shape=(h, w))
+def make_buffer(w, h, depth=4):
+    buf = array("B", [0x55] * (w*h*depth))
+    if depth == 4:
+        buf0 = memoryview(buf).cast("I", shape=(h, w))
+    else:
+        buf0 = memoryview(buf).cast("B", shape=(h, w))
     buf_p = c_void_p(buf.buffer_info()[0])
-    return buf, buf0, buf_p
+    return buf0, buf_p
 
 
 class BaseDebugWindow(PyBoyWindowPlugin):
@@ -377,7 +380,8 @@ def __init__(self, pyboy, mb, pyboy_argv, *, scale, title, width, height, pos_x,
         )
         self.window_id = sdl2.SDL_GetWindowID(self._window)
 
-        self.buf, self.buf0, self.buf_p = make_buffer(width, height)
+        self.buf0, self.buf_p = make_buffer(width, height)
+        self.buf0_attributes, _ = make_buffer(width, height, 1)
 
         self._sdlrenderer = sdl2.SDL_CreateRenderer(self._window, -1, sdl2.SDL_RENDERER_ACCELERATED)
         sdl2.SDL_RenderSetLogicalSize(self._sdlrenderer, width, height)
@@ -750,7 +754,7 @@ def post_tick(self):
                 self.buf0[y, x] = SPRITE_BACKGROUND
 
         for ly in range(144):
-            self.mb.lcd.renderer.scanline_sprites(self.mb.lcd, ly, self.buf0, True)
+            self.mb.lcd.renderer.scanline_sprites(self.mb.lcd, ly, self.buf0, self.buf0_attributes, True)
 
         self.draw_overlay()
         BaseDebugWindow.post_tick(self)
@@ -792,7 +796,7 @@ def __init__(self, *args, **kwargs):
         font_blob = "".join(line.strip() for line in font_lines[font_lines.index("BASE64DATA:\n") + 1:])
         font_bytes = zlib.decompress(b64decode(font_blob.encode()))
 
-        self.fbuf, self.fbuf0, self.fbuf_p = make_buffer(8, 16 * 256)
+        self.fbuf0, self.fbuf_p = make_buffer(8, 16 * 256)
         for y, b in enumerate(font_bytes):
             for x in range(8):
                 self.fbuf0[y, x] = 0xFFFFFFFF if ((0x80 >> x) & b) else 0x00000000

pyboy/plugins/screen_recorder.py

@@ -42,7 +42,7 @@ def handle_events(self, events):
     def post_tick(self):
         # Plugin: Screen Recorder
         if self.recording:
-            self.add_frame(self.pyboy.screen.image.convert(mode="RGBA"))
+            self.add_frame(self.pyboy.screen.image.copy())
 
     def add_frame(self, frame):
         # Pillow makes artifacts in the output, if we use 'RGB', which is PyBoy's default format

pyboy/utils.py

@@ -4,7 +4,7 @@
 #
 from enum import Enum
 
-STATE_VERSION = 9
+STATE_VERSION = 10
 
 ##############################################################
 # Buffer classes

tests/test_acid_cgb.py

@@ -26,8 +26,9 @@ def test_cgb_acid(cgb_acid_file):
         png_path.parents[0].mkdir(parents=True, exist_ok=True)
         image.save(png_path)
     else:
-        old_image = PIL.Image.open(png_path)
-        diff = PIL.ImageChops.difference(image.convert(mode="RGB"), old_image)
+        # Converting to RGB as ImageChops.difference cannot handle Alpha: https://github.com/python-pillow/Pillow/issues/4849
+        old_image = PIL.Image.open(png_path).convert("RGB")
+        diff = PIL.ImageChops.difference(image.convert("RGB"), old_image)
         if diff.getbbox() and not os.environ.get("TEST_CI"):
             image.show()
             old_image.show()

tests/test_acid_dmg.py

@@ -27,8 +27,9 @@ def test_dmg_acid(cgb, dmg_acid_file):
         png_path.parents[0].mkdir(parents=True, exist_ok=True)
         image.save(png_path)
     else:
-        old_image = PIL.Image.open(png_path)
-        diff = PIL.ImageChops.difference(image.convert(mode="RGB"), old_image)
+        # Converting to RGB as ImageChops.difference cannot handle Alpha: https://github.com/python-pillow/Pillow/issues/4849
+        old_image = PIL.Image.open(png_path).convert("RGB")
+        diff = PIL.ImageChops.difference(image.convert("RGB"), old_image)
         if diff.getbbox() and not os.environ.get("TEST_CI"):
             image.show()
             old_image.show()

tests/test_basics.py

@@ -219,8 +219,8 @@ def test_all_modes(cgb, _bootrom, frames, rom, any_rom_cgb, boot_cgb_rom):
             png_buf.write(b"".join([(x ^ 0b10011101).to_bytes(1, sys.byteorder) for x in data]))
         png_buf.seek(0)
 
-        old_image = PIL.Image.open(png_buf)
-        diff = PIL.ImageChops.difference(image.convert(mode="RGB"), old_image)
+        old_image = PIL.Image.open(png_buf).convert("RGB")
+        diff = PIL.ImageChops.difference(image.convert("RGB"), old_image)
         if diff.getbbox() and not os.environ.get("TEST_CI"):
             image.show()
             old_image.show()