Implemented hardware decoding

This implements hardware decoding continuing from the work of @rvillalba-novetta and @mikeboers in main...rvillalba-novetta:PyAV:hwaccel (and children commits)

av/__init__.py

@@ -17,6 +17,7 @@
 from av.bitstream import BitStreamFilterContext, bitstream_filters_available
 from av.codec.codec import Codec, codecs_available
 from av.codec.context import CodecContext
+from av.codec.hwaccel import HWConfig
 from av.container import open
 from av.format import ContainerFormat, formats_available
 from av.packet import Packet

av/__main__.py

@@ -6,6 +6,8 @@
 def main() -> None:
     parser = argparse.ArgumentParser()
     parser.add_argument("--codecs", action="store_true")
+    parser.add_argument("--hwdevices", action="store_true")
+    parser.add_argument("--hwconfigs", action="store_true")
     parser.add_argument("--version", action="store_true")
     args = parser.parse_args()
 
@@ -30,6 +32,16 @@ def main() -> None:
                 version = config["version"]
                 print(f"{libname:<13} {version[0]:3d}.{version[1]:3d}.{version[2]:3d}")
 
+    if args.hwdevices:
+        from av.codec.hwaccel import dump_hwdevices
+
+        dump_hwdevices()
+
+    if args.hwconfigs:
+        from av.codec.codec import dump_hwconfigs
+
+        dump_hwconfigs()
+
     if args.codecs:
         from av.codec.codec import dump_codecs
 

av/audio/codeccontext.pyx

@@ -3,13 +3,14 @@ cimport libav as lib
 from av.audio.format cimport AudioFormat, get_audio_format
 from av.audio.frame cimport AudioFrame, alloc_audio_frame
 from av.audio.layout cimport AudioLayout, get_audio_layout
+from av.codec.hwaccel cimport HWAccel
 from av.frame cimport Frame
 from av.packet cimport Packet
 
 
 cdef class AudioCodecContext(CodecContext):
-    cdef _init(self, lib.AVCodecContext *ptr, const lib.AVCodec *codec):
-        CodecContext._init(self, ptr, codec)
+    cdef _init(self, lib.AVCodecContext *ptr, const lib.AVCodec *codec, HWAccel hwaccel):
+        CodecContext._init(self, ptr, codec, hwaccel)
 
     cdef _prepare_frames_for_encode(self, Frame input_frame):
 

av/codec/codec.pxd

@@ -7,6 +7,8 @@ cdef class Codec:
     cdef const lib.AVCodecDescriptor *desc
     cdef readonly bint is_encoder
 
+    cdef tuple _hardware_configs
+
     cdef _init(self, name=?)
 
 

av/codec/codec.pyi

@@ -108,3 +108,4 @@ class codec_descriptor:
 codecs_available: set[str]
 
 def dump_codecs() -> None: ...
+def dump_hwconfigs() -> None: ...

av/codec/codec.pyx

@@ -1,4 +1,5 @@
 from av.audio.format cimport get_audio_format
+from av.codec.hwaccel cimport wrap_hwconfig
 from av.descriptor cimport wrap_avclass
 from av.utils cimport avrational_to_fraction
 from av.video.format cimport get_video_format
@@ -117,18 +118,25 @@ cdef class Codec:
         if self.is_encoder and lib.av_codec_is_decoder(self.ptr):
             raise RuntimeError("%s is both encoder and decoder.")
 
-    def create(self, kind = None):
+    def __repr__(self):
+        return f'<av.{self.__class__.__name__}({self.name!r}, {self.mode!r})>'
+
+    def create(self, kind = None, *args, **kwargs):
         """Create a :class:`.CodecContext` for this codec.
 
         :param str kind: Gives a hint to static type checkers for what exact CodecContext is used.
         """
         from .context import CodecContext
-        return CodecContext.create(self)
+        return CodecContext.create(self, *args, **kwargs)
 
     @property
     def is_decoder(self):
         return not self.is_encoder
 
+    @property
+    def mode(self):
+        return 'w' if self.is_encoder else 'r'
+
     @property
     def descriptor(self): return wrap_avclass(self.ptr.priv_class)
 
@@ -203,6 +211,23 @@ cdef class Codec:
             i += 1
         return ret
 
+    @property
+    def hardware_configs(self):
+        if self._hardware_configs:
+            return self._hardware_configs
+        ret = []
+        cdef int i = 0
+        cdef lib.AVCodecHWConfig *ptr
+        while True:
+            ptr = lib.avcodec_get_hw_config(self.ptr, i)
+            if not ptr:
+                break
+            ret.append(wrap_hwconfig(ptr))
+            i += 1
+        ret = tuple(ret)
+        self._hardware_configs = ret
+        return ret
+
     @property
     def properties(self):
         return self.desc.props
@@ -294,18 +319,16 @@ codec_descriptor = wrap_avclass(lib.avcodec_get_class())
 def dump_codecs():
     """Print information about available codecs."""
 
-    print(
-        """Codecs:
- D..... = Decoding supported
- .E.... = Encoding supported
- ..V... = Video codec
- ..A... = Audio codec
- ..S... = Subtitle codec
- ...I.. = Intra frame-only codec
- ....L. = Lossy compression
- .....S = Lossless compression
- ------"""
-    )
+    print('''Codecs:
+    D....  = Decoding supported
+    .E...  = Encoding supported
+    ..V..  = Video codec
+    ..A..  = Audio codec
+    ..S..  = Subtitle codec
+    ...I.  = Intra frame-only codec
+    ....L  = Lossless compression
+    .....H = Hardware decoding supported
+    ------''')
 
     for name in sorted(codecs_available):
         try:
@@ -323,17 +346,31 @@ def dump_codecs():
 
         try:
             print(
-                " %s%s%s%s%s%s %-18s %s"
+                "    %s%s%s%s%s%s %-18s %s"
                 % (
                     ".D"[bool(d_codec)],
                     ".E"[bool(e_codec)],
                     codec.type[0].upper(),
                     ".I"[codec.intra_only],
-                    ".L"[codec.lossy],
-                    ".S"[codec.lossless],
+                    ".L"[codec.lossless],
+                    ".H"[bool((d_codec or codec).hardware_configs)],
                     codec.name,
                     codec.long_name,
                 )
             )
         except Exception as e:
             print(f"...... {codec.name:<18} ERROR: {e}")
+
+def dump_hwconfigs():
+    print('Hardware configs:')
+    for name in sorted(codecs_available):
+        try:
+            codec = Codec(name, 'r')
+        except ValueError:
+            continue
+        configs = codec.hardware_configs
+        if not configs:
+            continue
+        print('   ', codec.name)
+        for config in configs:
+            print('       ', config)

av/codec/context.pxd

@@ -3,6 +3,7 @@ from libc.stdint cimport int64_t
 
 from av.bytesource cimport ByteSource
 from av.codec.codec cimport Codec
+from av.codec.hwaccel cimport HWAccel, HWAccelContext
 from av.frame cimport Frame
 from av.packet cimport Packet
 
@@ -18,11 +19,12 @@ cdef class CodecContext:
     cdef int stream_index
 
     cdef lib.AVCodecParserContext *parser
-    cdef _init(self, lib.AVCodecContext *ptr, const lib.AVCodec *codec)
+    cdef _init(self, lib.AVCodecContext *ptr, const lib.AVCodec *codec, HWAccel hwaccel)
 
     # Public API.
     cdef readonly bint is_open
     cdef readonly Codec codec
+    cdef readonly HWAccel hwaccel
     cdef public dict options
     cpdef open(self, bint strict=?)
 
@@ -31,6 +33,9 @@ cdef class CodecContext:
     cpdef decode(self, Packet packet=?)
     cpdef flush_buffers(self)
 
+    # Used by hardware-accelerated decode.
+    cdef HWAccelContext hwaccel_ctx
+
     # Used by both transcode APIs to setup user-land objects.
     # TODO: Remove the `Packet` from `_setup_decoded_frame` (because flushing packets
     # are bogus). It should take all info it needs from the context and/or stream.
@@ -49,10 +54,11 @@ cdef class CodecContext:
     cdef _send_packet_and_recv(self, Packet packet)
     cdef _recv_frame(self)
 
+    cdef _transfer_hwframe(self, Frame frame)
+
     # Implemented by children for the generic send/recv API, so we have the
     # correct subclass of Frame.
     cdef Frame _next_frame
     cdef Frame _alloc_next_frame(self)
 
-
-cdef CodecContext wrap_codec_context(lib.AVCodecContext*, const lib.AVCodec*)
+cdef CodecContext wrap_codec_context(lib.AVCodecContext*, const lib.AVCodec*, HWAccel hwaccel)

av/codec/context.pyi

@@ -92,3 +92,4 @@ class CodecContext:
         self, raw_input: bytes | bytearray | memoryview | None = None
     ) -> list[Packet]: ...
     def flush_buffers(self) -> None: ...
+    def is_hardware_accelerated(self) -> bool: ...

av/codec/context.pyx

@@ -18,7 +18,7 @@ from av.dictionary import Dictionary
 cdef object _cinit_sentinel = object()
 
 
-cdef CodecContext wrap_codec_context(lib.AVCodecContext *c_ctx, const lib.AVCodec *c_codec):
+cdef CodecContext wrap_codec_context(lib.AVCodecContext *c_ctx, const lib.AVCodec *c_codec, HWAccel hwaccel):
     """Build an av.CodecContext for an existing AVCodecContext."""
 
     cdef CodecContext py_ctx
@@ -35,7 +35,7 @@ cdef CodecContext wrap_codec_context(lib.AVCodecContext *c_ctx, const lib.AVCode
     else:
         py_ctx = CodecContext(_cinit_sentinel)
 
-    py_ctx._init(c_ctx, c_codec)
+    py_ctx._init(c_ctx, c_codec, hwaccel)
 
     return py_ctx
 
@@ -83,10 +83,10 @@ class Flags2(IntEnum):
 
 cdef class CodecContext:
     @staticmethod
-    def create(codec, mode=None):
+    def create(codec, mode=None, hwaccel=None):
         cdef Codec cy_codec = codec if isinstance(codec, Codec) else Codec(codec, mode)
         cdef lib.AVCodecContext *c_ctx = lib.avcodec_alloc_context3(cy_codec.ptr)
-        return wrap_codec_context(c_ctx, cy_codec.ptr)
+        return wrap_codec_context(c_ctx, cy_codec.ptr, hwaccel)
 
     def __cinit__(self, sentinel=None, *args, **kwargs):
         if sentinel is not _cinit_sentinel:
@@ -96,11 +96,12 @@ cdef class CodecContext:
         self.stream_index = -1  # This is set by the container immediately.
         self.is_open = False
 
-    cdef _init(self, lib.AVCodecContext *ptr, const lib.AVCodec *codec):
+    cdef _init(self, lib.AVCodecContext *ptr, const lib.AVCodec *codec, HWAccel hwaccel):
         self.ptr = ptr
         if self.ptr.codec and codec and self.ptr.codec != codec:
             raise RuntimeError("Wrapping CodecContext with mismatched codec.")
         self.codec = wrap_codec(codec if codec != NULL else self.ptr.codec)
+        self.hwaccel = hwaccel
 
         # Set reasonable threading defaults.
         self.ptr.thread_count = 0  # use as many threads as there are CPUs.
@@ -310,6 +311,9 @@ cdef class CodecContext:
 
         return packets
 
+    def is_hardware_accelerated(self):
+        return self.hwaccel_ctx is not None
+
     def _send_frame_and_recv(self, Frame frame):
         cdef Packet packet
 
@@ -359,10 +363,15 @@ cdef class CodecContext:
             return
         err_check(res)
 
+        frame = self._transfer_hwframe(frame)
+
         if not res:
             self._next_frame = None
             return frame
 
+    cdef _transfer_hwframe(self, Frame frame):
+        return frame
+
     cdef _recv_packet(self):
         cdef Packet packet = Packet()
 

av/codec/hwaccel.pxd

@@ -0,0 +1,22 @@
+cimport libav as lib
+
+from av.codec.codec cimport Codec
+
+
+cdef class HWConfig(object):
+    cdef object __weakref__
+    cdef lib.AVCodecHWConfig *ptr
+    cdef void _init(self, lib.AVCodecHWConfig *ptr)
+
+cdef HWConfig wrap_hwconfig(lib.AVCodecHWConfig *ptr)
+
+cdef class HWAccel(object):
+    cdef int _device_type
+    cdef str _device
+    cdef public bint allow_software_fallback
+    cdef public dict options
+
+cdef class HWAccelContext(HWAccel):
+    cdef readonly Codec codec
+    cdef readonly HWConfig config
+    cdef lib.AVBufferRef *ptr

av/codec/hwaccel.pyi

@@ -0,0 +1,46 @@
+from enum import IntEnum
+from typing import Sequence
+
+from av.codec.codec import Codec
+
+class HWDeviceType(IntEnum):
+    NONE = int
+    VDPAU = int
+    CUDA = int
+    VAAPI = int
+    DXVA2 = int
+    QSV = int
+    VIDEOTOOLBOX = int
+    D3D11VA = int
+    DRM = int
+    OPENCL = int
+    MEDIACODEC = int
+    VULKAN = int
+    D3D12VA = int
+
+class HWConfig(object):
+    def __init__(self, sentinel): ...
+    def __repr__(self): ...
+    @property
+    def device_type(self): ...
+    @property
+    def format(self): ...
+    @property
+    def methods(self): ...
+    @property
+    def is_supported(self): ...
+
+class HWAccel:
+    def __init__(
+        self,
+        device_type: str | HWDeviceType,
+        device: str | None = None,
+        allow_software_fallback: bool = True,
+        options=None,
+        **kwargs
+    ): ...
+    def create(self, codec: Codec): ...
+
+hwdevices_available: Sequence[str]
+
+def dump_hwdevices() -> None: ...