Skip to content

Commit

Permalink
Fix deferred operation handling error
Browse files Browse the repository at this point in the history 
when using deferred operation object in command vkcreateraytracingpipelines
, driver has two choices. One is driver proceed all workload in current
thread within this command, another choice is deferring this command to
other threads. The commit added handling to the second choice and fixed
related errors on pipeline reading and capture/playback shader group handle
error.

Change-Id: Id4704502714b866211555d007e09dcefa69a4138
  • Loading branch information
@mizhen
mizhen committed Jul 14, 2023
1 parent 7f60dc4 commit 7b11eaf
Show file tree
Hide file tree
Showing 22 changed files with 421 additions and 119 deletions.
13 changes: 13 additions & 0 deletions framework/decode/vulkan_consumer_base.h
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
/*
** Copyright (c) 2018-2020 Valve Corporation
** Copyright (c) 2018-2020 LunarG, Inc.
** Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
**
** Permission is hereby granted, free of charge, to any person obtaining a
** copy of this software and associated documentation files (the "Software"),
Expand Down Expand Up @@ -170,6 +171,18 @@ class VulkanConsumerBase
format::HandleId descriptorUpdateTemplate,
DescriptorUpdateTemplateDecoder* pData)
{}

virtual void Process_vkCreateRayTracingPipelinesKHR(
const ApiCallInfo& call_info,
VkResult returnValue,
format::HandleId device,
format::HandleId deferredOperation,
format::HandleId pipelineCache,
uint32_t createInfoCount,
StructPointerDecoder<Decoded_VkRayTracingPipelineCreateInfoKHR>* pCreateInfos,
StructPointerDecoder<Decoded_VkAllocationCallbacks>* pAllocator,
HandlePointerDecoder<VkPipeline>* pPipelines)
{}
};

GFXRECON_END_NAMESPACE(decode)
Expand Down
25 changes: 25 additions & 0 deletions framework/decode/vulkan_export_json_consumer_base.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/*
** Copyright (c) 2022-2023 LunarG, Inc.
** Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
**
** Permission is hereby granted, free of charge, to any person obtaining a
** copy of this software and associated documentation files (the "Software"),
Expand Down Expand Up @@ -616,6 +617,30 @@ void VulkanExportJsonConsumerBase::Process_vkUpdateDescriptorSetWithTemplateKHR(
WriteBlockEnd();
}

void VulkanExportJsonConsumerBase::Process_vkCreateRayTracingPipelinesKHR(
const ApiCallInfo& call_info,
VkResult returnValue,
format::HandleId device,
format::HandleId deferredOperation,
format::HandleId pipelineCache,
uint32_t createInfoCount,
StructPointerDecoder<Decoded_VkRayTracingPipelineCreateInfoKHR>* pCreateInfos,
StructPointerDecoder<Decoded_VkAllocationCallbacks>* pAllocator,
HandlePointerDecoder<VkPipeline>* pPipelines)
{
nlohmann::ordered_json& jdata = WriteApiCallStart(call_info, "vkCreateRayTracingPipelinesKHR");
FieldToJson(jdata[NameReturn()], returnValue, json_options_);
auto& args = jdata[NameArgs()];
HandleToJson(args["device"], device, json_options_);
HandleToJson(args["deferredOperation"], deferredOperation, json_options_);
HandleToJson(args["pipelineCache"], pipelineCache, json_options_);
FieldToJson(args["createInfoCount"], createInfoCount, json_options_);
FieldToJson(args["pCreateInfos"], pCreateInfos, json_options_);
FieldToJson(args["pAllocator"], pAllocator, json_options_);
HandleToJson(args["pPipelines"], pPipelines, json_options_);
WriteBlockEnd();
}

void VulkanExportJsonConsumerBase::WriteBlockStart()
{
json_data_.clear(); // < Dominates profiling (1/2).
Expand Down
12 changes: 12 additions & 0 deletions framework/decode/vulkan_export_json_consumer_base.h
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/*
** Copyright (c) 2022-2023 LunarG, Inc.
** Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
**
** Permission is hereby granted, free of charge, to any person obtaining a
** copy of this software and associated documentation files (the "Software"),
Expand Down Expand Up @@ -177,6 +178,17 @@ class VulkanExportJsonConsumerBase : public VulkanConsumer, public AnnotationHan
format::HandleId descriptorUpdateTemplate,
DescriptorUpdateTemplateDecoder* pData) override;

virtual void Process_vkCreateRayTracingPipelinesKHR(
const ApiCallInfo& call_info,
VkResult returnValue,
format::HandleId device,
format::HandleId deferredOperation,
format::HandleId pipelineCache,
uint32_t createInfoCount,
StructPointerDecoder<Decoded_VkRayTracingPipelineCreateInfoKHR>* pCreateInfos,
StructPointerDecoder<Decoded_VkAllocationCallbacks>* pAllocator,
HandlePointerDecoder<VkPipeline>* pPipelines) override;

/// @brief Convert annotations, which are simple {type:enum, key:string, value:string} objects.
virtual void ProcessAnnotation(uint64_t block_index,
format::AnnotationType type,
Expand Down
5 changes: 4 additions & 1 deletion framework/decode/vulkan_object_info.h
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/*
** Copyright (c) 2019-2022 LunarG, Inc.
** Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
**
** Permission is hereby granted, free of charge, to any person obtaining a
** copy of this software and associated documentation files (the "Software"),
Expand Down Expand Up @@ -438,11 +439,13 @@ struct FramebufferInfo : public VulkanObjectInfo<VkFramebuffer>

struct DeferredOperationKHRInfo : public VulkanObjectInfo<VkDeferredOperationKHR>
{
VkResult join_state{ VK_NOT_READY };
bool pending_state{ false };

// Record CreateRayTracingPipelinesKHR parameters for safety.
std::vector<VkRayTracingPipelineCreateInfoKHR> record_modified_create_infos;
std::vector<std::vector<VkRayTracingShaderGroupCreateInfoKHR>> record_modified_pgroups;
std::vector<VkPipeline> replayPipelines;
std::vector<format::HandleId> capturePipelines;
};

struct VideoSessionKHRInfo : VulkanObjectInfo<VkVideoSessionKHR>
Expand Down
104 changes: 100 additions & 4 deletions framework/decode/vulkan_replay_consumer_base.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
/*
** Copyright (c) 2018-2020 Valve Corporation
** Copyright (c) 2018-2023 LunarG, Inc.
** Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
**
** Permission is hereby granted, free of charge, to any person obtaining a
** copy of this software and associated documentation files (the "Software"),
Expand Down Expand Up @@ -6025,9 +6026,14 @@ VkResult VulkanReplayConsumerBase::OverrideCreateRayTracingPipelinesKHR(

if (deferred_operation_info)
{
deferred_operation_info->join_state = VK_NOT_READY;
deferred_operation_info->pending_state = true;
deferred_operation_info->record_modified_create_infos.clear();
deferred_operation_info->record_modified_pgroups.clear();
deferred_operation_info->replayPipelines.resize(createInfoCount);
deferred_operation_info->capturePipelines.resize(createInfoCount);
memcpy(deferred_operation_info->capturePipelines.data(),
pPipelines->GetPointer(),
createInfoCount * sizeof(VkPipeline));
}

if (device_info->property_feature_info.feature_rayTracingPipelineShaderGroupHandleCaptureReplay)
Expand Down Expand Up @@ -6092,7 +6098,33 @@ VkResult VulkanReplayConsumerBase::OverrideCreateRayTracingPipelinesKHR(
createInfoCount,
modified_create_infos.data(),
in_pAllocator,
out_pPipelines);
deferred_operation_info->replayPipelines.data());
if ((result == VK_SUCCESS) || (result == VK_OPERATION_NOT_DEFERRED_KHR) ||
(result == VK_PIPELINE_COMPILE_REQUIRED_EXT))
{
// The above return values mean the command is not deferred and driver will finish all workload in current
// thread. Therefore the created pipelines can be read and copied to out_pPipelines which will be
// referenced later.
//
// Note:
// Some pipelines might actually fail creation if the return value is VK_PIPELINE_COMPILE_REQUIRED_EXT.
// These failed pipelines will generate VK_NULL_HANDLE.
//
// If the return value is VK_OPERATION_DEFERRED_KHR, it means the command is deferred, and thus pipeline
// creation is not finished. Subsequent handling will be done by
// vkDeferredOperationJoinKHR/vkGetDeferredOperationResultKHR after pipeline creation is finished.
memcpy(
out_pPipelines, deferred_operation_info->replayPipelines.data(), createInfoCount * sizeof(VkPipeline));

if (deferred_operation_info)
{
// Eventhough vkCreateRayTracingPipelinesKHR was called with a valid deferred operation object, the
// driver may opt to not defer the command. In this case, set pending_state flag to false to skip
// vkDeferredOperationJoinKHR handling.
deferred_operation_info->pending_state = false;
}
}

if (deferred_operation_info)
{
deferred_operation_info->record_modified_create_infos = std::move(modified_create_infos);
Expand Down Expand Up @@ -6122,8 +6154,9 @@ VkResult VulkanReplayConsumerBase::OverrideDeferredOperationJoinKHR(PFN_vkDeferr
const DeviceInfo* device_info,
DeferredOperationKHRInfo* deferred_operation_info)
{
if (deferred_operation_info->join_state == VK_SUCCESS)
if (deferred_operation_info->pending_state == false)
{
// The deferred operation object has no deferred command or its deferred command has been finished.
return VK_SUCCESS;
}

Expand Down Expand Up @@ -6161,9 +6194,19 @@ VkResult VulkanReplayConsumerBase::OverrideDeferredOperationJoinKHR(PFN_vkDeferr
j.get();
}

deferred_operation_info->join_state = VK_SUCCESS;
AddHandles<PipelineInfo>(device_info->capture_id,
deferred_operation_info->capturePipelines.data(),
deferred_operation_info->capturePipelines.size(),
deferred_operation_info->replayPipelines.data(),
deferred_operation_info->replayPipelines.size(),
&VulkanObjectInfoTable::AddPipelineInfo);

deferred_operation_info->pending_state = false;
deferred_operation_info->record_modified_create_infos.clear();
deferred_operation_info->record_modified_pgroups.clear();
deferred_operation_info->capturePipelines.clear();
deferred_operation_info->replayPipelines.clear();

return VK_SUCCESS;
}

Expand Down Expand Up @@ -6754,5 +6797,58 @@ void VulkanReplayConsumerBase::Process_vkUpdateDescriptorSetWithTemplateKHR(cons
in_device, in_descriptorSet, in_descriptorUpdateTemplate, pData->GetPointer());
}

void VulkanReplayConsumerBase::Process_vkCreateRayTracingPipelinesKHR(
const ApiCallInfo& call_info,
VkResult returnValue,
format::HandleId device,
format::HandleId deferredOperation,
format::HandleId pipelineCache,
uint32_t createInfoCount,
StructPointerDecoder<Decoded_VkRayTracingPipelineCreateInfoKHR>* pCreateInfos,
StructPointerDecoder<Decoded_VkAllocationCallbacks>* pAllocator,
HandlePointerDecoder<VkPipeline>* pPipelines)
{
auto in_device = GetObjectInfoTable().GetDeviceInfo(device);
auto in_deferredOperation = GetObjectInfoTable().GetDeferredOperationKHRInfo(deferredOperation);
auto in_pipelineCache = GetObjectInfoTable().GetPipelineCacheInfo(pipelineCache);
MapStructArrayHandles(pCreateInfos->GetMetaStructPointer(), pCreateInfos->GetLength(), GetObjectInfoTable());

if (!pPipelines->IsNull())
{
pPipelines->SetHandleLength(createInfoCount);
}

std::vector<PipelineInfo> handle_info(createInfoCount);

for (size_t i = 0; i < createInfoCount; ++i)
{
pPipelines->SetConsumerData(i, &handle_info[i]);
}

VkResult replay_result =
OverrideCreateRayTracingPipelinesKHR(GetDeviceTable(in_device->handle)->CreateRayTracingPipelinesKHR,
returnValue,
in_device,
in_deferredOperation,
in_pipelineCache,
createInfoCount,
pCreateInfos,
pAllocator,
pPipelines);
CheckResult("vkCreateRayTracingPipelinesKHR", returnValue, replay_result);

if ((replay_result == VK_SUCCESS) || (replay_result == VK_OPERATION_NOT_DEFERRED_KHR) ||
(replay_result == VK_PIPELINE_COMPILE_REQUIRED_EXT))
{
AddHandles<PipelineInfo>(device,
pPipelines->GetPointer(),
pPipelines->GetLength(),
pPipelines->GetHandlePointer(),
createInfoCount,
std::move(handle_info),
&VulkanObjectInfoTable::AddPipelineInfo);
}
}

GFXRECON_END_NAMESPACE(decode)
GFXRECON_END_NAMESPACE(gfxrecon)
11 changes: 11 additions & 0 deletions framework/decode/vulkan_replay_consumer_base.h
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
/*
** Copyright (c) 2018-2020 Valve Corporation
** Copyright (c) 2018-2022 LunarG, Inc.
** Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
**
** Permission is hereby granted, free of charge, to any person obtaining a
** copy of this software and associated documentation files (the "Software"),
Expand Down Expand Up @@ -178,6 +179,16 @@ class VulkanReplayConsumerBase : public VulkanConsumer
format::HandleId descriptorSet,
format::HandleId descriptorUpdateTemplate,
DescriptorUpdateTemplateDecoder* pData) override;
virtual void Process_vkCreateRayTracingPipelinesKHR(
const ApiCallInfo& call_info,
VkResult returnValue,
format::HandleId device,
format::HandleId deferredOperation,
format::HandleId pipelineCache,
uint32_t createInfoCount,
StructPointerDecoder<Decoded_VkRayTracingPipelineCreateInfoKHR>* pCreateInfos,
StructPointerDecoder<Decoded_VkAllocationCallbacks>* pAllocator,
HandlePointerDecoder<VkPipeline>* pPipelines);

protected:
const VulkanObjectInfoTable& GetObjectInfoTable() const { return object_info_table_; }
Expand Down
53 changes: 35 additions & 18 deletions framework/encode/custom_vulkan_api_call_encoders.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
/*
** Copyright (c) 2018-2020 Valve Corporation
** Copyright (c) 2018-2020 LunarG, Inc.
** Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
**
** Permission is hereby granted, free of charge, to any person obtaining a
** copy of this software and associated documentation files (the "Software"),
Expand Down Expand Up @@ -652,24 +653,40 @@ VKAPI_ATTR VkResult VKAPI_CALL CreateRayTracingPipelinesKHR(VkDevice
omit_output_data = true;
}

auto encoder = VulkanCaptureManager::Get()->BeginTrackedApiCallCapture(
format::ApiCallId::ApiCall_vkCreateRayTracingPipelinesKHR);
if (encoder)
{
encoder->EncodeHandleValue<DeviceWrapper>(device);
encoder->EncodeHandleValue<DeferredOperationKHRWrapper>(deferredOperation);
encoder->EncodeHandleValue<PipelineCacheWrapper>(pipelineCache);
encoder->EncodeUInt32Value(createInfoCount);
EncodeStructArray(encoder, pCreateInfos, createInfoCount);
EncodeStructPtr(encoder, pAllocator);
encoder->EncodeHandleArray<PipelineWrapper>(pPipelines, createInfoCount, omit_output_data);
encoder->EncodeEnumValue(result);
VulkanCaptureManager::Get()
->EndGroupCreateApiCallCapture<VkDevice,
VkDeferredOperationKHR,
PipelineWrapper,
VkRayTracingPipelineCreateInfoKHR>(
result, device, deferredOperation, createInfoCount, pPipelines, pCreateInfos);
auto device_wrapper = GetWrapper<DeviceWrapper>(device);
if ((result != VK_OPERATION_DEFERRED_KHR) ||
(!device_wrapper->property_feature_info.feature_rayTracingPipelineShaderGroupHandleCaptureReplay))
{
// If the operation is not deferred by driver. or the system doesn't support
// rayTracingPipelineShaderGroupHandleCaptureReplay, we don't need to delay writing the block of
// vkCreateRayTracingPipelinesKHR to the capture file.
//
// If the operation is deferred by driver and the system supports
// rayTracingPipelineShaderGroupHandleCaptureReplay, we will need to make sure opaque capture replay data for
// pipeline shader group handles are ready when calling vkCreateRayTracingPipelinesKHR during replay. However,
// opaque capture replay data for pipeline shader group handles needs pipeline creation to be finished so that
// vkGetRayTracingCaptureReplayShaderGroupHandlesKHR may be used. For this reason, we will delay the writing of
// vkCreateRayTracingPipelinesKHR block to capture file until the deferred operation is finished.

auto encoder = VulkanCaptureManager::Get()->BeginTrackedApiCallCapture(
format::ApiCallId::ApiCall_vkCreateRayTracingPipelinesKHR);
if (encoder)
{
encoder->EncodeHandleValue<DeviceWrapper>(device);
encoder->EncodeHandleValue<DeferredOperationKHRWrapper>(deferredOperation);
encoder->EncodeHandleValue<PipelineCacheWrapper>(pipelineCache);
encoder->EncodeUInt32Value(createInfoCount);
EncodeStructArray(encoder, pCreateInfos, createInfoCount);
EncodeStructPtr(encoder, pAllocator);
encoder->EncodeHandleArray<PipelineWrapper>(pPipelines, createInfoCount, omit_output_data);
encoder->EncodeEnumValue(result);
VulkanCaptureManager::Get()
->EndGroupCreateApiCallCapture<VkDevice,
VkDeferredOperationKHR,
PipelineWrapper,
VkRayTracingPipelineCreateInfoKHR>(
result, device, deferredOperation, createInfoCount, pPipelines, pCreateInfos);
}
}

CustomEncoderPostCall<format::ApiCallId::ApiCall_vkCreateRayTracingPipelinesKHR>::Dispatch(
Expand Down
Loading

0 comments on commit 7b11eaf

Please sign in to comment.