From 95da0d2f042c1f4f1a1f42774089bad4f922e085 Mon Sep 17 00:00:00 2001
From: thesword53 <thesword53@gmail.com>
Date: Wed, 8 Jan 2025 23:54:34 +0100
Subject: [PATCH 1/2] Fixed some potential deadlocks

---
 src/vabackend.c | 22 ++++++++++++++++++++--
 src/vabackend.h |  1 +
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/src/vabackend.c b/src/vabackend.c
index 2ae386c..b2213e3 100644
--- a/src/vabackend.c
+++ b/src/vabackend.c
@@ -428,6 +428,14 @@ static void* resolveSurfaces(void *param) {
             ctx->surfaceQueueReadIdx = 0;
         }
 
+        if (surface->decodeFailed) {
+            pthread_mutex_lock(&surface->mutex);
+            surface->resolving = 0;
+            pthread_cond_signal(&surface->cond);
+            pthread_mutex_unlock(&surface->mutex);
+            continue;
+        }
+
         CUdeviceptr deviceMemory = (CUdeviceptr) NULL;
         unsigned int pitch = 0;
 
@@ -440,6 +448,10 @@ static void* resolveSurfaces(void *param) {
 
         //LOG("Mapping surface %d", surface->pictureIdx);
         if (CHECK_CUDA_RESULT(cv->cuvidMapVideoFrame(ctx->decoder, surface->pictureIdx, &deviceMemory, &pitch, &procParams))) {
+            pthread_mutex_lock(&surface->mutex);
+            surface->resolving = 0;
+            pthread_cond_signal(&surface->cond);
+            pthread_mutex_unlock(&surface->mutex);
             continue;
         }
         //LOG("Mapped surface %d to %p (%d)", surface->pictureIdx, (void*)deviceMemory, pitch);
@@ -1334,11 +1346,16 @@ static VAStatus nvEndPicture(
     nvCtx->bitstreamBuffer.size = 0;
     nvCtx->sliceOffsets.size = 0;
 
+    CHECK_CUDA_RESULT_RETURN(cu->cuCtxPushCurrent(drv->cudaContext), VA_STATUS_ERROR_OPERATION_FAILED);
     CUresult result = cv->cuvidDecodePicture(nvCtx->decoder, picParams);
+    CHECK_CUDA_RESULT_RETURN(cu->cuCtxPopCurrent(NULL), VA_STATUS_ERROR_OPERATION_FAILED);
+
+    VAStatus status = VA_STATUS_SUCCESS;
+
     if (result != CUDA_SUCCESS)
     {
         LOG("cuvidDecodePicture failed: %d", result);
-        return VA_STATUS_ERROR_DECODING_ERROR;
+        status = VA_STATUS_ERROR_DECODING_ERROR;
     }
     //LOG("Decoded frame successfully to idx: %d (%p)", picParams->CurrPicIdx, nvCtx->renderTarget);
 
@@ -1347,6 +1364,7 @@ static VAStatus nvEndPicture(
     surface->context = nvCtx;
     surface->topFieldFirst = !picParams->bottom_field_flag;
     surface->secondField = picParams->second_field;
+    surface->decodeFailed = status != VA_STATUS_SUCCESS;
 
     //TODO check we're not overflowing the queue
     pthread_mutex_lock(&nvCtx->resolveMutex);
@@ -1359,7 +1377,7 @@ static VAStatus nvEndPicture(
     //Wake up the resolve thread
     pthread_cond_signal(&nvCtx->resolveCondition);
 
-    return VA_STATUS_SUCCESS;
+    return status;
 }
 
 static VAStatus nvSyncSurface(
diff --git a/src/vabackend.h b/src/vabackend.h
index b1ec781..1841d41 100644
--- a/src/vabackend.h
+++ b/src/vabackend.h
@@ -67,6 +67,7 @@ typedef struct
     int                     resolving;
     pthread_mutex_t         mutex;
     pthread_cond_t          cond;
+    bool                    decodeFailed;
 } NVSurface;
 
 typedef enum

From ae80f52f81479a4117535c2aeb765e4f2e97821e Mon Sep 17 00:00:00 2001
From: thesword53 <thesword53@gmail.com>
Date: Thu, 9 Jan 2025 22:14:52 +0100
Subject: [PATCH 2/2] Fix H.264 video decode error when seeking/looping

---
 src/vabackend.c | 14 +++++++++-----
 src/vabackend.h |  1 +
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/vabackend.c b/src/vabackend.c
index b2213e3..c7b3d9c 100644
--- a/src/vabackend.c
+++ b/src/vabackend.c
@@ -1046,17 +1046,17 @@ static VAStatus nvCreateContext(
     if (num_render_targets) {
         // Update the decoder configuration to match the passed in surfaces.
         NVSurface *surface = (NVSurface *) getObjectPtr(drv, render_targets[0]);
+        if (!surface) {
+            return VA_STATUS_ERROR_INVALID_PARAMETER;
+        }
         cfg->surfaceFormat = surface->format;
         cfg->chromaFormat = surface->chromaFormat;
         cfg->bitDepth = surface->bitDepth;
     }
 
-    if (drv->surfaceCount <= 1 && num_render_targets == 0) {
-        LOG("0/1 surfaces have been passed to vaCreateContext, this might cause errors. Setting surface count to 32");
-        num_render_targets = 32;
-    }
+     // Setting to maximun value if num_render_targets == 0 to prevent picture index overflow as additional surfaces can be created after calling nvCreateContext
+    int surfaceCount = num_render_targets > 0 ? num_render_targets : 32;
 
-    int surfaceCount = drv->surfaceCount > 1 ? drv->surfaceCount : num_render_targets;
     if (surfaceCount > 32) {
         LOG("Application requested %d surface(s), limiting to 32. This may cause issues.", surfaceCount);
         surfaceCount = 32;
@@ -1100,6 +1100,7 @@ static VAStatus nvCreateContext(
     nvCtx->width = picture_width;
     nvCtx->height = picture_height;
     nvCtx->codec = selectedCodec;
+    nvCtx->surfaceCount = surfaceCount;
 
     pthread_mutexattr_t attrib;
     pthread_mutexattr_init(&attrib);
@@ -1277,6 +1278,9 @@ static VAStatus nvBeginPicture(
 
     //if this surface hasn't been used before, give it a new picture index
     if (surface->pictureIdx == -1) {
+        if (nvCtx->currentPictureId == nvCtx->surfaceCount) {
+            return VA_STATUS_ERROR_MAX_NUM_EXCEEDED;
+        }
         surface->pictureIdx = nvCtx->currentPictureId++;
     }
 
diff --git a/src/vabackend.h b/src/vabackend.h
index 1841d41..ac39e7f 100644
--- a/src/vabackend.h
+++ b/src/vabackend.h
@@ -179,6 +179,7 @@ typedef struct _NVContext
     int                 surfaceQueueWriteIdx;
     bool                exiting;
     pthread_mutex_t     surfaceCreationMutex;
+    int                 surfaceCount;
 } NVContext;
 
 typedef struct