Fix engine classes serialization

src/xrCore/Save/MemoryBuffer.cpp

@@ -13,9 +13,9 @@ bool CMemoryChunk::Write(const void* data, size_t size)
 	if (!CanWrite(size)) {
 		return false;
 	}
-	memcpy(this->data, data, size);
+	memcpy(this->data + count, data, size);
 	count += size;
-	return false;
+	return true;
 }
 
 CMemoryBuffer::CMemoryBuffer()

src/xrCore/Save/MemoryBuffer.h

@@ -27,7 +27,7 @@ class CReservedMemory
 class CMemoryChunk
 {
 public:
-	static constexpr size_t ChunkSize = 16 * 1024;
+	static constexpr size_t ChunkSize = 2 * 1024;
 
 private:
 	BYTE	data[ChunkSize];
@@ -51,6 +51,12 @@ class CMemoryChunk
 		return Write(data, sizeof(T));
 	}
 
+	template<>
+	bool Write(IWriter* data) {
+		data->w((BYTE*)&this->data, count);
+		return false;
+	}
+
 };
 
 class CMemoryBuffer 
@@ -85,4 +91,12 @@ class CMemoryBuffer
 		Write(data.c_str(), data.size());
 		return true;
 	}
+
+	template<>
+	bool Write(IWriter* data) {
+		for (const auto& elem : Chunks) {
+			elem->Write(data);
+		}
+		return true;
+	}
 };

src/xrCore/Save/SaveChunk.cpp

@@ -16,12 +16,7 @@ CSaveChunk::~CSaveChunk()
 void CSaveChunk::Write(CMemoryBuffer& Buffer)
 {
 	Buffer.Write((u8)ESaveVariableType::t_chunkStart);
-	if (CSaveManager::GetInstance().TestFlag(CSaveManager::ESaveManagerFlagsGeneral::EUseStringOptimization)) {
-		Buffer.Write(_chunkName);
-	}
-	else {
-		Buffer.Write(_chunkName);
-	}
+	CSaveManager::GetInstance().ConditionalWriteString(_chunkName, Buffer);
 	for (const auto& elem : _subchunks) {
 		elem.second->Write(Buffer);
 	}

src/xrCore/Save/SaveManager.cpp

@@ -4,9 +4,9 @@
 
 CSaveManager::CSaveManager() 
 {
-	SetFlag(ESaveManagerFlagsGeneral::EUseStringOptimization, false);
-	SetFlag(ESaveManagerFlagsGeneral::EUseIntOptimization, false);
-	SetFlag(ESaveManagerFlagsGeneral::EUseBoolOptimization, false);
+	SetFlag(ESaveManagerFlagsGeneral::EUseStringOptimization, true);
+	SetFlag(ESaveManagerFlagsGeneral::EUseIntOptimization, true);
+	SetFlag(ESaveManagerFlagsGeneral::EUseBoolOptimization, true);
 	SetFlag(ESaveManagerFlagsGeneral::EHasExtraControlFlags, false);
 }
 
@@ -43,6 +43,10 @@ CSaveObjectSave* CSaveManager::BeginSave()
 void CSaveManager::WriteSavedData(const string_path& to_file)
 {
 	SaveWriter = FS.w_open(to_file);
+	Buffers.Init();
+	StringsHashesMap = xr_make_unique<xr_map<u32, xr_vector<shared_str>>>();
+	BoolQueue = xr_make_unique<xr_queue<bool>>();
+	CompileData();
 	WriteHeader();
 	if (TestFlag(ESaveManagerFlagsGeneral::EUseStringOptimization))
 	{
@@ -52,23 +56,123 @@ void CSaveManager::WriteSavedData(const string_path& to_file)
 	{
 		WriteBools();
 	}
+	WriteData();
+	StringsHashesMap.reset();
+	BoolQueue.reset();
+	Buffers.Clear();
+	FS.w_close(SaveWriter);
+}
+
+void CSaveManager::ConditionalWriteString(shared_str Value, CMemoryBuffer& buffer)
+{
+	if (TestFlag(CSaveManager::ESaveManagerFlagsGeneral::EUseStringOptimization)) {
+		auto StringKey = crc32(Value.c_str(), Value.size());
+		u32 StringVecID = 0;
+		auto it = StringsHashesMap->find(StringKey);
+		if (it == StringsHashesMap->end()) {
+			StringsHashesMap->emplace(StringKey, RStringVec{ Value });
+		}
+		else {
+			bool Contains = false;
+			for (u32 i = 0; i < it->second.size(); ++i) {
+				if (Value == it->second[i]) {
+					Contains = true;
+					StringVecID = i;
+					break;
+				}
+			}
+			if (!Contains) {
+				StringVecID = it->second.size();
+				it->second.emplace_back(Value);
+			}
+		}
+		u64 StringRefID = ((u64)StringKey << 32) | StringVecID;
+		buffer.Write(StringRefID);
+	}
+	else {
+		buffer.Write(Value);
+	}
+}
+
+void CSaveManager::ConditionalWriteBool(bool& Value, CMemoryBuffer& buffer)
+{
+	if (TestFlag(CSaveManager::ESaveManagerFlagsGeneral::EUseBoolOptimization)) 
+	{
+		BoolQueue->push(&Value);
+		++BoolsNum;
+	}
+	else {
+		buffer.Write(Value);
+	}
 }
 
 void CSaveManager::WriteHeader()
 {
-	SaveWriter->w_u8(ControlFlagsDefault.flags);
+	Buffers.BufferHeader->Write(ESaveVariableType::t_chunk);
+	Buffers.BufferHeader->Write(ControlFlagsDefault.flags);
+	Buffers.BufferHeader->Write(SaveWriter);
 }
 
 void CSaveManager::WriteStrings()
 {
+	Buffers.BufferStrings->Write(ESaveVariableType::t_chunk);
+	Buffers.BufferStrings->Write(StringsHashesMap->size());
+	Buffers.BufferStrings->Write(ESaveVariableType::t_array);
+	for (const auto& elem : *StringsHashesMap.get()) {
+		Buffers.BufferStrings->Write(elem.first);
+		Buffers.BufferStrings->Write(ESaveVariableType::t_array);
+		Buffers.BufferStrings->Write(elem.second.size());
+		for (const auto& elem2 : elem.second) {
+			Buffers.BufferStrings->Write(elem2);
+		}
+	}
+	Buffers.BufferStrings->Write(SaveWriter);
 }
 
 void CSaveManager::WriteBools()
 {
+	Buffers.BufferBools->Write(ESaveVariableType::t_chunk);
+	Buffers.BufferBools->Write(BoolsNum);
+	Flags8 Flags;
+	u8 WrittenFlags = 0;
+	while (!BoolQueue->empty()) {
+		auto Ptr = BoolQueue->front();
+		Flags.set(1 << WrittenFlags, Ptr);
+		++WrittenFlags;
+		if (WrittenFlags == 8) {
+			Buffers.BufferBools->Write(Flags.get());
+			WrittenFlags = 0;
+			Flags.zero();
+		}
+		BoolQueue->pop();
+	}
+	if (WrittenFlags != 8) {
+		Buffers.BufferBools->Write(Flags.get());
+	}
+	Buffers.BufferBools->Write(SaveWriter);
 }
 
 void CSaveManager::WriteData()
 {
-	CMemoryBuffer buffer;
-	SaveData->Write(buffer);
+	Buffers.BufferGeneral->Write(SaveWriter);
+}
+
+void CSaveManager::CompileData()
+{
+	SaveData->Write(Buffers.BufferGeneral);
+}
+
+inline void CSaveManager::SMemoryBuffers::Init() {
+	VERIFY(!(BufferHeader || BufferStrings || BufferBools || BufferGeneral));
+	BufferHeader = new CMemoryBuffer();
+	BufferStrings = new CMemoryBuffer();
+	BufferBools = new CMemoryBuffer();
+	BufferGeneral = new CMemoryBuffer();
+}
+
+inline void CSaveManager::SMemoryBuffers::Clear() {
+	xr_delete(BufferHeader);
+	xr_delete(BufferStrings);
+	xr_delete(BufferBools);
+	xr_delete(BufferGeneral);
 }

src/xrCore/Save/SaveManager.h

@@ -1,12 +1,68 @@
 #pragma once
 #include "SaveObject.h"
 
+/*
+New save file structure:
+
+1) Header chunk: settings of the save
+1.1) Firstly - chunk mark
+1.2) Some flags
+1.3) EUseStringOptimization flag - gather all strings in different chunk and replace their occurence with u64 (u32 string hash + u32 index for collisions)
++ Shrink save memory because writes similar strings once
+- Need more time to process
+1.4) EUseIntOptimization flag - attempt to write all integers with fewer amount of bytes. For example, if u64 has value of 200, then it'll be writen as u8, but still will be read as u64
++ Shrink save memory because uses less bytes for each integer
+- Need more time to process
+1.5) EUseBoolOptimization flag - write all bools in different chunk, representing each bool as individual bit instead of byte. All bool values from general chunk would be romeved and will be restores during read.
++ Shrink save memory because uses only one bit per bool instead od byte per bool
+- Strongly depends on order of writing bools in memory, could be broken if somehow order of bools in memory would be changed
+1.6) EHasExtraControlFlags flag - reserved for showing if additional settings chunk present
+
+2) (Optional) Additional header chunk - write your required settings data. Please, left a single flag if where would be other additional header chunks futher
+
+3) (Optional) String chunk - write compressed strings
+3.1) Firstly - chunk mark
+3.2) Write map "string hash - string container" as array
+3.3) Write each string container as array of strings
+
+4) (Optional) Bools chunk - write compressed bools
+4.1) Firstly - chunk mark
+4.2) Amount of u8, containing data.
+4.3) Sequence of u8, each bit represents each bool at corresponding position
+
+5) General chunk - chunk with serialized data
+5.1) Serialized tree. Saved recursevly.
+5.2) Firstly - chunk mark
+5.3) After that, chunk name
+5.4) Then, subchunks (start from point 5.2)
+5.5) Variables in this chunk
+5.6) Chunk end mark
+*/
+
 class XRCORE_API CSaveManager
 {
 	CSaveManager();
 
 	CSaveObjectSave* SaveData = nullptr;
 	IWriter* SaveWriter = nullptr;
+	xr_unique_ptr<xr_map<u32, xr_vector<shared_str>>> StringsHashesMap;
+	xr_unique_ptr<xr_queue<bool>> BoolQueue;
+	u64 BoolsNum = 0;
+
+public:
+	struct SMemoryBuffers {
+		CMemoryBuffer* BufferHeader = nullptr;
+		CMemoryBuffer* BufferStrings = nullptr;
+		CMemoryBuffer* BufferBools = nullptr;
+		CMemoryBuffer* BufferGeneral = nullptr;
+
+		void Init();
+		void Clear();
+
+	}; 
+
+private:
+	SMemoryBuffers Buffers;
 	Flags8 ControlFlagsDefault;
 	Flags8 ControlFlagsRead;
 
@@ -15,6 +71,8 @@ class XRCORE_API CSaveManager
 	void WriteBools();
 	void WriteData();
 
+	void CompileData();
+
 public:
 
 	enum class ESaveManagerFlagsGeneral : u8 {
@@ -38,4 +96,7 @@ class XRCORE_API CSaveManager
 	CSaveObjectSave* BeginSave();
 	void WriteSavedData(const string_path& to_file);
 
+	void ConditionalWriteString(shared_str Value, CMemoryBuffer& buffer);
+	void ConditionalWriteBool(bool& Value, CMemoryBuffer& buffer);
+
 };

src/xrCore/Save/SaveObject.cpp

@@ -1,5 +1,6 @@
 #include "stdafx.h"
 #include "SaveObject.h"
+#include "SaveManager.h"
 
 CSaveObject::CSaveObject()
 {
@@ -62,6 +63,11 @@ void CSaveObjectSave::BeginChunk(shared_str ChunkName)
 	_chunkStack.push(_chunkStack.top()->BeginChunk(ChunkName));
 }
 
+void CSaveObjectSave::Write(CMemoryBuffer* buffer)
+{
+	_rootChunk->Write(*buffer);
+}
+
 CSaveObjectLoad::CSaveObjectLoad()
 {
 	_rootChunk = new CSaveChunk("Root");

src/xrCore/Save/SaveObject.h

@@ -16,6 +16,7 @@ class XRCORE_API CSaveObject {
 	CSaveChunk* GetCurrentChunk();
 	virtual void BeginChunk(shared_str ChunkName) = 0;
 	void EndChunk();
+
 };
 
 class XRCORE_API CSaveObjectSave: public CSaveObject {
@@ -24,6 +25,8 @@ class XRCORE_API CSaveObjectSave: public CSaveObject {
 	CSaveObjectSave(CSaveChunk* Root);
 
 	virtual void BeginChunk(shared_str ChunkName) override;
+
+	void Write(CMemoryBuffer* buffer);
 };
 
 class XRCORE_API CSaveObjectLoad: public CSaveObject {