Skip to content

Commit

Permalink
Support multiple simulataneous API capture managers
Browse files Browse the repository at this point in the history 
Split common and api specific options from isA to refsA relationship.

For single API sessions should be exactly equivalent.
  • Loading branch information
@jzulauf-lunarg
jzulauf-lunarg committed May 6, 2024
1 parent b8b598c commit 7ecaee7
Show file tree
Hide file tree
Showing 7 changed files with 597 additions and 308 deletions.
184 changes: 135 additions & 49 deletions framework/encode/capture_manager.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,11 +51,10 @@ std::mutex CaptureManager::ThreadData::count
format::ThreadId CaptureManager::ThreadData::thread_count_ = 0;
std::unordered_map<uint64_t, format::ThreadId> CaptureManager::ThreadData::id_map_;

uint32_t CaptureManager::instance_count_ = 0;
CaptureManager* CaptureManager::singleton_;
std::mutex CaptureManager::instance_lock_;
thread_local std::unique_ptr<CaptureManager::ThreadData> CaptureManager::thread_data_;
CaptureManager::ApiCallMutexT CaptureManager::api_call_mutex_;
std::function<void()> CaptureManager::delete_instance_func_ = nullptr;

std::atomic<format::HandleId> CaptureManager::unique_id_counter_{ format::kNullHandleId };

Expand Down Expand Up @@ -88,8 +87,8 @@ format::ThreadId CaptureManager::ThreadData::GetThreadId()
return id;
}

CaptureManager::CaptureManager(format::ApiFamilyId api_family) :
api_family_(api_family), force_file_flush_(false), timestamp_filename_(true),
CaptureManager::CaptureManager() :
force_file_flush_(false), timestamp_filename_(true),
memory_tracking_mode_(CaptureSettings::MemoryTrackingMode::kPageGuard), page_guard_align_buffer_sizes_(false),
page_guard_track_ahb_memory_(false), page_guard_unblock_sigsegv_(false), page_guard_signal_handler_watcher_(false),
page_guard_memory_mode_(kMemoryModeShadowInternal), trim_enabled_(false),
Expand All @@ -112,90 +111,128 @@ CaptureManager::~CaptureManager()
util::Log::Release();
}

bool CaptureManager::CreateInstance(std::function<CaptureManager*()> GetInstanceFunc,
std::function<void()> NewInstanceFunc,
std::function<void()> DeleteInstanceFunc)
bool CaptureManager::CreateInstance(ApiCaptureManagerBase* api_capture_singleton,
const std::function<void()>& destroyer)
{
bool success = true;
std::lock_guard<std::mutex> instance_lock(instance_lock_);

if (instance_count_ == 0)
if (!singleton_)
{
assert(GetInstanceFunc() == nullptr);
singleton_ = new CaptureManager();
}

return singleton_->LockedCreateInstance(api_capture_singleton, destroyer);
}

bool CaptureManager::LockedCreateInstance(ApiCaptureManagerBase* api_capture_singleton,
const std::function<void()>& destroyer)
{
bool success = true;

// Create new instance of capture manager.
instance_count_ = 1;
NewInstanceFunc();
delete_instance_func_ = DeleteInstanceFunc;
if (instance_count_ == 0)
{
if (std::atexit(CaptureManager::AtExit))
{
GFXRECON_LOG_WARNING("Failed registering atexit");
}

assert(GetInstanceFunc() != nullptr);

// Initialize logging to report only errors (to stderr).
util::Log::Settings stderr_only_log_settings;
stderr_only_log_settings.min_severity = util::Log::kErrorSeverity;
stderr_only_log_settings.output_errors_to_stderr = true;
util::Log::Init(stderr_only_log_settings);

// Get capture settings which can be different per capture manager.
CaptureSettings settings(GetInstanceFunc()->GetDefaultTraceSettings());
// NOTE: FIRST Api Instance is used for settings -- actual multiple simulatenous API support will need to
// resolve. Get capture settings which can be different per capture manager.
default_settings_ = api_capture_singleton->GetDefaultTraceSettings();
capture_settings_ = api_capture_singleton->GetDefaultTraceSettings();

// Load log settings.
CaptureSettings::LoadLogSettings(&settings);
CaptureSettings::LoadLogSettings(&capture_settings_);

// Reinitialize logging with values retrieved from settings.
util::Log::Release();
util::Log::Init(settings.GetLogSettings());
util::Log::Init(capture_settings_.GetLogSettings());

// Load all settings with final logging settings active.
CaptureSettings::LoadSettings(&settings);
CaptureSettings::LoadSettings(&capture_settings_);

GFXRECON_LOG_INFO("Initializing GFXReconstruct capture layer");
GFXRECON_LOG_INFO(" GFXReconstruct Version %s", GFXRECON_PROJECT_VERSION_STRING);

CaptureSettings::TraceSettings trace_settings = settings.GetTraceSettings();
CaptureSettings::TraceSettings trace_settings = capture_settings_.GetTraceSettings();
std::string base_filename = trace_settings.capture_file;

// Initialize capture manager with default settings.
success = GetInstanceFunc()->Initialize(base_filename, trace_settings);
success = Initialize(base_filename, trace_settings);
if (!success)
{
GFXRECON_LOG_FATAL("Failed to initialize CaptureManager");
}
}

// Add the api capture managers to the list
GFXRECON_ASSERT(api_capture_singleton != nullptr);
auto manager_it = api_capture_managers_.find(api_capture_singleton);
if (manager_it == api_capture_managers_.end())
{
// Add the Api Capture Manager to this list, and point it back to this common manager.
auto inserted =
api_capture_managers_.insert(std::make_pair(api_capture_singleton, ApiInstanceRecord{ 1, destroyer }));
api_capture_singleton->SetCommonManager(this);
GFXRECON_ASSERT(inserted.second);
manager_it = inserted.first;

// NOTE: moved here from CaptureTracker::Initialize... DRY'r than putting it into the API specific
// CreateInstances. For actual multiple simulatenous API support we need to ensure all API capture manager
// state trackers are in the correct state given the differing settings that may be present.
if ((capture_mode_ & kModeTrack) == kModeTrack)
{
api_capture_singleton->CreateStateTracker();
}
}
else
{
assert(GetInstanceFunc() != nullptr);
++instance_count_;
++manager_it->second.count;
}

++instance_count_;

GFXRECON_LOG_DEBUG("CaptureManager::CreateInstance(): Current instance count is %u", instance_count_);

return success;
}

void CaptureManager::DestroyInstance(std::function<const CaptureManager*()> GetInstanceFunc)
void CaptureManager::DestroyInstance(ApiCaptureManagerBase* api_capture_manager)
{
std::lock_guard<std::mutex> instance_lock(instance_lock_);

if (GetInstanceFunc() != nullptr)
GFXRECON_ASSERT(instance_count_ > 0);

auto manager_it = api_capture_managers_.find(api_capture_manager);
GFXRECON_ASSERT(manager_it != api_capture_managers_.end());

if (manager_it != api_capture_managers_.end())
{
assert(instance_count_ > 0);
GFXRECON_ASSERT(manager_it->second.count > 0);
--manager_it->second.count;

if (manager_it->second.count == 0)
{
GFXRECON_ASSERT(manager_it->first);
manager_it->second.destroyer();
api_capture_managers_.erase(manager_it);
}
--instance_count_;
GFXRECON_LOG_DEBUG("CaptureManager::DestroyInstance(): Current instance count is %u", instance_count_);

if (instance_count_ == 0)
{
assert(delete_instance_func_);
delete_instance_func_();
delete_instance_func_ = nullptr;
assert(GetInstanceFunc() == nullptr);
// NOTE: This delete is deleting this, so don't touch the object after this is done.
GFXRECON_ASSERT(this == singleton_);
delete singleton_;
singleton_ = nullptr;
}

GFXRECON_LOG_DEBUG("CaptureManager::DestroyInstance(): Current instance count is %u", instance_count_);
}
}

Expand Down Expand Up @@ -426,11 +463,6 @@ bool CaptureManager::Initialize(std::string base_filename, const CaptureSettings
trace_settings.page_guard_signal_handler_watcher_max_restores,
mem_prot_mode);
}

if ((capture_mode_ & kModeTrack) == kModeTrack)
{
CreateStateTracker();
}
}
else
{
Expand All @@ -440,6 +472,24 @@ bool CaptureManager::Initialize(std::string base_filename, const CaptureSettings
return success;
}

CaptureManager::ThreadData* CaptureManager::GetThreadData()
{
if (!thread_data_)
{
thread_data_ = std::make_unique<ThreadData>();
}
return thread_data_.get();
}

bool CaptureManager::IsCaptureModeTrack() const
{
return (GetCaptureMode() & kModeTrack) == kModeTrack;
}
bool CaptureManager::IsCaptureModeWrite() const
{
return (GetCaptureMode() & kModeWrite) == kModeWrite;
}

ParameterEncoder* CaptureManager::InitApiCallCapture(format::ApiCallId call_id)
{
auto thread_data = GetThreadData();
Expand Down Expand Up @@ -640,7 +690,11 @@ void CaptureManager::CheckContinueCaptureForWriteMode(uint32_t current_boundary_
trim_enabled_ = false;
trim_boundary_ = CaptureSettings::TrimBoundary::kUnknown;
capture_mode_ = kModeDisabled;
DestroyStateTracker();
// Clean up all of the capture manager's state trackers
for (auto& manager_it : api_capture_managers_)
{
manager_it.first->DestroyStateTracker();
}
compressor_ = nullptr;
}
else if (trim_ranges_[trim_current_range_].first == current_boundary_count)
Expand Down Expand Up @@ -911,7 +965,8 @@ void CaptureManager::ActivateTrimming()
auto thread_data = GetThreadData();
assert(thread_data != nullptr);

WriteTrackedState(file_stream_.get(), thread_data->thread_id_);
for (auto& manager : api_capture_managers_)
manager.first->WriteTrackedState(file_stream_.get(), thread_data->thread_id_);
}

void CaptureManager::DeactivateTrimming()
Expand Down Expand Up @@ -954,6 +1009,21 @@ void CaptureManager::BuildOptionList(const format::EnabledOptions& enable
option_list->push_back({ format::FileOption::kCompressionType, enabled_options.compression_type });
}

format::ApiFamilyId CaptureManager::GetApiFamily() const
{
const format::ApiFamilyId none_id = format::ApiFamilyId::ApiFamily_None;
format::ApiFamilyId id = none_id;
const size_t manager_count = api_capture_managers_.size();
if (manager_count == 1)
{
id = api_capture_managers_.cbegin()->first->GetApiFamily();
}
else if (manager_count > 1)
{
id = format::ApiFamilyId::ApiFamily_Multiple;
}
return id;
}
void CaptureManager::WriteDisplayMessageCmd(const char* message)
{
if ((capture_mode_ & kModeWrite) == kModeWrite)
Expand All @@ -964,8 +1034,11 @@ void CaptureManager::WriteDisplayMessageCmd(const char* message)

message_cmd.meta_header.block_header.type = format::BlockType::kMetaDataBlock;
message_cmd.meta_header.block_header.size = format::GetMetaDataBlockBaseSize(message_cmd) + message_length;

// NOTE: The message output may vary based on the ordering of Instance Creation/Destruction relative to
// message generation. Need to addres for actual multiple simulatenous API support.
message_cmd.meta_header.meta_data_id =
format::MakeMetaDataId(api_family_, format::MetaDataType::kDisplayMessageCommand);
format::MakeMetaDataId(GetApiFamily(), format::MetaDataType::kDisplayMessageCommand);
message_cmd.thread_id = thread_data->thread_id_;

CombineAndWriteToFile({ { &message_cmd, sizeof(message_cmd) }, { message, message_length } });
Expand All @@ -982,7 +1055,7 @@ void CaptureManager::WriteExeFileInfo(const gfxrecon::util::filepath::FileInfo&
exe_info_header.meta_header.block_header.type = format::BlockType::kMetaDataBlock;
exe_info_header.meta_header.block_header.size = format::GetMetaDataBlockBaseSize(exe_info_header);
exe_info_header.meta_header.meta_data_id =
format::MakeMetaDataId(api_family_, format::MetaDataType::kExeFileInfoCommand);
format::MakeMetaDataId(GetApiFamily(), format::MetaDataType::kExeFileInfoCommand);
exe_info_header.thread_id = thread_data->thread_id_;

WriteToFile(&exe_info_header, sizeof(exe_info_header));
Expand Down Expand Up @@ -1022,7 +1095,7 @@ void CaptureManager::WriteResizeWindowCmd(format::HandleId surface_id, uint32_t
resize_cmd.meta_header.block_header.type = format::BlockType::kMetaDataBlock;
resize_cmd.meta_header.block_header.size = format::GetMetaDataBlockBaseSize(resize_cmd);
resize_cmd.meta_header.meta_data_id =
format::MakeMetaDataId(api_family_, format::MetaDataType::kResizeWindowCommand);
format::MakeMetaDataId(GetApiFamily(), format::MetaDataType::kResizeWindowCommand);
resize_cmd.thread_id = thread_data->thread_id_;

resize_cmd.surface_id = surface_id;
Expand All @@ -1049,7 +1122,7 @@ void CaptureManager::WriteFillMemoryCmd(format::HandleId memory_id, uint64_t off

fill_cmd.meta_header.block_header.type = format::BlockType::kMetaDataBlock;
fill_cmd.meta_header.meta_data_id =
format::MakeMetaDataId(api_family_, format::MetaDataType::kFillMemoryCommand);
format::MakeMetaDataId(GetApiFamily(), format::MetaDataType::kFillMemoryCommand);
fill_cmd.thread_id = thread_data->thread_id_;
fill_cmd.memory_id = memory_id;
fill_cmd.memory_offset = offset;
Expand Down Expand Up @@ -1092,7 +1165,7 @@ void CaptureManager::WriteFillMemoryCmd(format::HandleId memory_id, uint64_t off

void CaptureManager::WriteCreateHeapAllocationCmd(uint64_t allocation_id, uint64_t allocation_size)
{
if ((GetCaptureMode() & kModeWrite) == kModeWrite)
if (IsCaptureModeWrite())
{
format::CreateHeapAllocationCommand allocation_cmd;

Expand All @@ -1102,7 +1175,7 @@ void CaptureManager::WriteCreateHeapAllocationCmd(uint64_t allocation_id, uint64
allocation_cmd.meta_header.block_header.type = format::BlockType::kMetaDataBlock;
allocation_cmd.meta_header.block_header.size = format::GetMetaDataBlockBaseSize(allocation_cmd);
allocation_cmd.meta_header.meta_data_id =
format::MakeMetaDataId(api_family_, format::MetaDataType::kCreateHeapAllocationCommand);
format::MakeMetaDataId(GetApiFamily(), format::MetaDataType::kCreateHeapAllocationCommand);
allocation_cmd.thread_id = thread_data->thread_id_;
allocation_cmd.allocation_id = allocation_id;
allocation_cmd.allocation_size = allocation_size;
Expand Down Expand Up @@ -1151,9 +1224,22 @@ void CaptureManager::WriteToFile(const void* data, size_t size)
thread_data->block_index_ = block_index_.load();
}

void CaptureManager::AtExit()
{
if (CaptureManager::singleton_)
{
for (auto& api_capture_manager : singleton_->api_capture_managers_)
{
api_capture_manager.second.destroyer();
}
delete CaptureManager::singleton_;
CaptureManager::singleton_ = nullptr;
}
}

void CaptureManager::WriteCaptureOptions(std::string& operation_annotation)
{
CaptureSettings::TraceSettings default_settings = GetDefaultTraceSettings();
CaptureSettings::TraceSettings default_settings = default_settings_.GetTraceSettings();
std::string buffer;

if (force_file_flush_ != default_settings.force_flush)
Expand Down Expand Up @@ -1251,7 +1337,7 @@ void CaptureManager::WriteCaptureOptions(std::string& operation_annotation)
operation_annotation += "\n }";
}

CaptureSettings::TraceSettings CaptureManager::GetDefaultTraceSettings()
CaptureSettings::TraceSettings ApiCaptureManagerBase::GetDefaultTraceSettings()
{
// Return default trace settings.
return CaptureSettings::TraceSettings();
Expand Down
Loading

0 comments on commit 7ecaee7

Please sign in to comment.