- Global codebook related fixes

- Minor fix to basisu_frontend::optimize_selector_codebook() so the new_selector_cluster_indices array is initialized correctly. (The backend doesn't use this array so no harm was being done.)

basisu_tool.cpp

@@ -37,7 +37,7 @@
 using namespace basisu;
 using namespace buminiz;
 
-#define BASISU_TOOL_VERSION "1.13"
+#define BASISU_TOOL_VERSION "1.14"
 
 enum tool_mode
 {
@@ -120,7 +120,7 @@ static void print_usage()
 		" -bench: UASTC benchmark mode, for development only\n"
 		" -resample_factor X: Resample all input textures by scale factor X using a box filter\n"
 		" -no_sse: Forbid all SSE instruction set usage\n"
-		" -validate_etc1s: Validate internal ETC1S compressor's data structures.\n"
+		" -validate_etc1s: Validate internal ETC1S compressor's data structures during compression (slower, intended for development).\n"
 		"\n"
 		"Mipmap generation options:\n"
 		" -mipmap: Generate mipmaps for each source image\n"
@@ -417,6 +417,7 @@ class command_line_params
 				m_comp_params.m_compute_stats = true;
 			else if (strcasecmp(pArg, "-gen_global_codebooks") == 0)
 			{
+				// TODO
 			}
 			else if (strcasecmp(pArg, "-use_global_codebooks") == 0)
 			{
@@ -781,6 +782,7 @@ struct basis_data
 	uint8_vec m_file_data;
 	basist::basisu_transcoder m_transcoder;
 };
+
 static basis_data *load_basis_file(const char *pInput_filename, basist::etc1_global_selector_codebook &sel_codebook, bool force_etc1s)
 {
 	basis_data* p = new basis_data(sel_codebook);
@@ -821,6 +823,7 @@ static basis_data *load_basis_file(const char *pInput_filename, basist::etc1_glo
 	}
 	return p;
 }
+
 static bool compress_mode(command_line_params &opts)
 {
 	basist::etc1_global_selector_codebook sel_codebook(basist::g_global_selector_cb_size, basist::g_global_selector_cb);
@@ -848,13 +851,18 @@ static bool compress_mode(command_line_params &opts)
 		error_printf("No input files to process!\n");
 		return false;
 	}
+
 	basis_data* pGlobal_codebook_data = nullptr;
 	if (opts.m_etc1s_use_global_codebooks_file.size())
 	{
 		pGlobal_codebook_data = load_basis_file(opts.m_etc1s_use_global_codebooks_file.c_str(), sel_codebook, true);
 		if (!pGlobal_codebook_data)
 			return false;
+
+		printf("Loaded global codebooks from .basis file \"%s\"\n", opts.m_etc1s_use_global_codebooks_file.c_str());
+
 #if 0
+		// Development/test code. TODO: Remove.
 		basis_data* pGlobal_codebook_data2 = load_basis_file("xmen_1024.basis", sel_codebook, true);
 		const basist::basisu_lowlevel_etc1s_transcoder &ta = pGlobal_codebook_data->m_transcoder.get_lowlevel_etc1s_decoder();
 		const basist::basisu_lowlevel_etc1s_transcoder &tb = pGlobal_codebook_data2->m_transcoder.get_lowlevel_etc1s_decoder();
@@ -1131,8 +1139,13 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 	{
 		pGlobal_codebook_data = load_basis_file(opts.m_etc1s_use_global_codebooks_file.c_str(), sel_codebook, true);
 		if (!pGlobal_codebook_data)
+		{
+			error_printf("Failed loading global codebook data from file \"%s\"\n", opts.m_etc1s_use_global_codebooks_file.c_str());
 			return false;
+		}
+		printf("Loaded global codebooks from file \"%s\"\n", opts.m_etc1s_use_global_codebooks_file.c_str());
 	}
+
 	if (!opts.m_input_filenames.size())
 	{
 		error_printf("No input files to process!\n");
@@ -1295,9 +1308,31 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 		if (opts.m_mode == cInfo)
 		{
 			if (pCSV_file) fclose(pCSV_file);
+			delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 			return true;
 		}
 
+		if ((fileinfo.m_etc1s) && (fileinfo.m_selector_codebook_size == 0) && (fileinfo.m_endpoint_codebook_size == 0))
+		{
+			// File is ETC1S and uses global codebooks - make sure we loaded one
+			if (!pGlobal_codebook_data)
+			{
+				error_printf("ETC1S file uses global codebooks, but none were loaded (see the -use_global_codebooks option)\n");
+				if (pCSV_file) fclose(pCSV_file);
+				delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
+				return false;
+			}
+
+			if ((pGlobal_codebook_data->m_transcoder.get_lowlevel_etc1s_decoder().get_endpoints().size() != fileinfo.m_total_endpoints) ||
+				(pGlobal_codebook_data->m_transcoder.get_lowlevel_etc1s_decoder().get_selectors().size() != fileinfo.m_total_selectors))
+			{
+				error_printf("Supplied global codebook is not compatible with this file\n");
+				if (pCSV_file) fclose(pCSV_file);
+				delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
+				return false;
+			}
+		}
+
 		interval_timer tm;
 		tm.start();
 

encoder/basisu_backend.cpp

@@ -739,6 +739,7 @@ namespace basisu
 
 				}  // block_x
 			} // block_y
+
 			for (uint32_t block_y = 0; block_y < num_blocks_y; block_y++)
 			{
 				for (uint32_t block_x = 0; block_x < num_blocks_x; block_x++)

encoder/basisu_basis_file.cpp

@@ -68,8 +68,8 @@ namespace basisu
 		m_header.m_total_endpoints = encoder_output.m_num_endpoints;
 		if (!encoder_output.m_uses_global_codebooks)
 		{
-		m_header.m_endpoint_cb_file_ofs = m_endpoint_cb_file_ofs;
-		m_header.m_endpoint_cb_file_size = (uint32_t)encoder_output.m_endpoint_palette.size();
+			m_header.m_endpoint_cb_file_ofs = m_endpoint_cb_file_ofs;
+			m_header.m_endpoint_cb_file_size = (uint32_t)encoder_output.m_endpoint_palette.size();
 		}
 		else
 		{
@@ -79,8 +79,8 @@ namespace basisu
 		m_header.m_total_selectors = encoder_output.m_num_selectors;
 		if (!encoder_output.m_uses_global_codebooks)
 		{
-		m_header.m_selector_cb_file_ofs = m_selector_cb_file_ofs;
-		m_header.m_selector_cb_file_size = (uint32_t)encoder_output.m_selector_palette.size();
+			m_header.m_selector_cb_file_ofs = m_selector_cb_file_ofs;
+			m_header.m_selector_cb_file_size = (uint32_t)encoder_output.m_selector_palette.size();
 		}
 		else
 		{
@@ -152,16 +152,16 @@ namespace basisu
 
 		if (!encoder_output.m_uses_global_codebooks)
 		{
-		if (encoder_output.m_endpoint_palette.size())
-		{
-			assert(m_comp_data.size() == m_endpoint_cb_file_ofs);
-			append_vector(m_comp_data, reinterpret_cast<const uint8_t*>(&encoder_output.m_endpoint_palette[0]), encoder_output.m_endpoint_palette.size());
-		}
+			if (encoder_output.m_endpoint_palette.size())
+			{
+				assert(m_comp_data.size() == m_endpoint_cb_file_ofs);
+				append_vector(m_comp_data, reinterpret_cast<const uint8_t*>(&encoder_output.m_endpoint_palette[0]), encoder_output.m_endpoint_palette.size());
+			}
 
-		if (encoder_output.m_selector_palette.size())
-		{
-			assert(m_comp_data.size() == m_selector_cb_file_ofs);
-			append_vector(m_comp_data, reinterpret_cast<const uint8_t*>(&encoder_output.m_selector_palette[0]), encoder_output.m_selector_palette.size());
+			if (encoder_output.m_selector_palette.size())
+			{
+				assert(m_comp_data.size() == m_selector_cb_file_ofs);
+				append_vector(m_comp_data, reinterpret_cast<const uint8_t*>(&encoder_output.m_selector_palette[0]), encoder_output.m_selector_palette.size());
 			}
 		}
 
@@ -227,9 +227,9 @@ namespace basisu
 			}
 			else
 			{
-			m_endpoint_cb_file_ofs = m_slice_descs_file_ofs + sizeof(basist::basis_slice_desc) * (uint32_t)slice_descs.size();
-			m_selector_cb_file_ofs = m_endpoint_cb_file_ofs + (uint32_t)encoder_output.m_endpoint_palette.size();
-			m_tables_file_ofs = m_selector_cb_file_ofs + (uint32_t)encoder_output.m_selector_palette.size();
+				m_endpoint_cb_file_ofs = m_slice_descs_file_ofs + sizeof(basist::basis_slice_desc) * (uint32_t)slice_descs.size();
+				m_selector_cb_file_ofs = m_endpoint_cb_file_ofs + (uint32_t)encoder_output.m_endpoint_palette.size();
+				m_tables_file_ofs = m_selector_cb_file_ofs + (uint32_t)encoder_output.m_selector_palette.size();
 			}
 			m_first_image_file_ofs = m_tables_file_ofs + (uint32_t)encoder_output.m_slice_image_tables.size();
 		}

encoder/basisu_frontend.cpp

@@ -297,7 +297,7 @@ namespace basisu
 					}
 				}
 			}
-
+						
 			optimize_selector_codebook();
 
 			if (m_params.m_debug_stats)
@@ -415,6 +415,7 @@ namespace basisu
 		}
 #endif
 
+		// HACK HACK
 		const uint32_t NUM_PASSES = 3;
 		for (uint32_t pass = 0; pass < NUM_PASSES; pass++)
 		{
@@ -503,7 +504,7 @@ namespace basisu
 				const uint32_t last_index = minimum<uint32_t>(m_total_blocks, first_index + N);
 
 #ifndef __EMSCRIPTEN__
-				m_params.m_pJob_pool->add_job([this, first_index, last_index, pass] {
+				m_params.m_pJob_pool->add_job([this, first_index, last_index] {
 #endif
 
 					for (uint32_t block_index = first_index; block_index < last_index; block_index++)
@@ -733,12 +734,17 @@ namespace basisu
 			if (m_optimized_cluster_selector_global_cb_ids.size())
 				new_optimized_cluster_selector_global_cb_ids[i] = m_optimized_cluster_selector_global_cb_ids[new_to_old[i]];
 
-			if (m_selector_cluster_block_indices.size())
-				new_selector_cluster_indices[i] = m_selector_cluster_block_indices[new_to_old[i]];
+			//if (m_selector_cluster_block_indices.size())
+			//	new_selector_cluster_indices[i] = m_selector_cluster_block_indices[new_to_old[i]];
 
 			if (m_selector_cluster_uses_global_cb.size())
 				new_selector_cluster_uses_global_cb[i] = m_selector_cluster_uses_global_cb[new_to_old[i]];
 		}
+
+		for (uint32_t i = 0; i < m_block_selector_cluster_index.size(); i++)
+		{
+			new_selector_cluster_indices[m_block_selector_cluster_index[i]].push_back(i);
+		}
 
 		m_optimized_cluster_selectors.swap(new_optimized_cluster_selectors);
 		m_optimized_cluster_selector_global_cb_ids.swap(new_optimized_cluster_selector_global_cb_ids);

transcoder/basisu_transcoder.cpp

@@ -7949,9 +7949,7 @@ namespace basist
 		}
 
 		approx_move_to_front selector_history_buf(m_selector_history_buf_size);
-
-		const uint32_t SELECTOR_HISTORY_BUF_FIRST_SYMBOL_INDEX = (uint32_t)m_local_selectors.size();
-		const uint32_t SELECTOR_HISTORY_BUF_RLE_SYMBOL_INDEX = m_selector_history_buf_size + SELECTOR_HISTORY_BUF_FIRST_SYMBOL_INDEX;
+
 		uint32_t cur_selector_rle_count = 0;
 
 		decoder_etc_block block;
@@ -7991,6 +7989,9 @@ namespace basist
 			return false;
 		}
 
+		const uint32_t SELECTOR_HISTORY_BUF_FIRST_SYMBOL_INDEX = (uint32_t)selectors.size();
+		const uint32_t SELECTOR_HISTORY_BUF_RLE_SYMBOL_INDEX = m_selector_history_buf_size + SELECTOR_HISTORY_BUF_FIRST_SYMBOL_INDEX;
+
 		for (uint32_t block_y = 0; block_y < num_blocks_y; block_y++)
 		{
 			const uint32_t cur_block_endpoint_pred_array = block_y & 1;

transcoder/basisu_transcoder.h

@@ -377,11 +377,13 @@ namespace basist
 		uint32_t m_total_header_size;
 
 		uint32_t m_total_selectors;
-		uint32_t m_selector_codebook_ofs;
+		// will be 0 for UASTC or if the file uses global codebooks
+		uint32_t m_selector_codebook_ofs; 
 		uint32_t m_selector_codebook_size;
 
 		uint32_t m_total_endpoints;
-		uint32_t m_endpoint_codebook_ofs;
+		// will be 0 for UASTC or if the file uses global codebooks
+		uint32_t m_endpoint_codebook_ofs; 
 		uint32_t m_endpoint_codebook_size;
 
 		uint32_t m_tables_ofs;

transcoder/basisu_transcoder_internal.h

@@ -20,8 +20,8 @@
 #pragma warning (disable: 4127) //  conditional expression is constant
 #endif
 
-#define BASISD_LIB_VERSION 113
-#define BASISD_VERSION_STRING "01.13"
+#define BASISD_LIB_VERSION 114
+#define BASISD_VERSION_STRING "01.14"
 
 #ifdef _DEBUG
 #define BASISD_BUILD_DEBUG