skyculture-converter: deduplicate images

10110111 · Jan 17, 2025 · 74709f6 · 74709f6
1 parent 02771df
commit 74709f6
Show file tree

Hide file tree

Showing 2 changed files with 65 additions and 13 deletions.
diff --git a/util/skyculture-converter/DescriptionOldLoader.cpp b/util/skyculture-converter/DescriptionOldLoader.cpp
@@ -699,12 +699,22 @@ void DescriptionOldLoader::loadTranslationsOfNames(const QString& poBaseDir, con
 	}
 }
 
-void DescriptionOldLoader::locateAllInlineImages(const QString& html)
+void DescriptionOldLoader::locateAndRelocateAllInlineImages(QString& html, const bool saveToRefs)
 {
 	for(auto matches = htmlGeneralImageRegex.globalMatch(html); matches.hasNext(); )
 	{
 		const auto& match = matches.next();
-		imageHRefs.emplace_back(match.captured(1));
+		auto path = match.captured(1);
+		auto updatedPath = path;
+		if(!path.startsWith("illustrations/"))
+		{
+			updatedPath = "illustrations/" + path;
+			const auto imgTag = match.captured(0);
+			const auto updatedImgTag = QString(imgTag).replace(path, updatedPath);
+			html.replace(imgTag, updatedImgTag);
+		}
+		if(saveToRefs)
+			imageHRefs.emplace_back(path, updatedPath);
 	}
 }
 
@@ -721,8 +731,8 @@ void DescriptionOldLoader::load(const QString& inDir, const QString& poBaseDir,
 		qCritical().noquote() << "Failed to open file" << englishDescrPath;
 		return;
 	}
-	const auto html = englishDescrFile.readAll();
-	locateAllInlineImages(html);
+	QString html = englishDescrFile.readAll();
+	locateAndRelocateAllInlineImages(html, true);
 	qDebug() << "Processing English description...";
 	markdown = convertHTMLToMarkdown(html, fullerConversionToMarkdown, footnotesToRefs, convertOrderedLists);
 
@@ -827,7 +837,9 @@ void DescriptionOldLoader::load(const QString& inDir, const QString& poBaseDir,
 			continue;
 		}
 		qDebug().nospace() << "Processing description for locale " << locale << "...";
-		auto trMD0 = convertHTMLToMarkdown(file.readAll(), fullerConversionToMarkdown, footnotesToRefs, convertOrderedLists);
+		QString localizedHTML = file.readAll();
+		locateAndRelocateAllInlineImages(localizedHTML, false);
+		auto trMD0 = convertHTMLToMarkdown(localizedHTML, fullerConversionToMarkdown, footnotesToRefs, convertOrderedLists);
 		const auto translationMD = trMD0.replace(QRegularExpression("<notr>([^<]+)</notr>"), "\\1");
 		const auto translatedSections = splitToSections(translationMD);
 		if(translatedSections.size() != englishSections.size())
@@ -993,23 +1005,55 @@ bool DescriptionOldLoader::dumpMarkdown(const QString& outDir) const
 
 	for(const auto& img : imageHRefs)
 	{
-		const auto imgInPath = inputDir+"/"+img;
-		if(!QFileInfo(imgInPath).exists())
+		const auto imgInPath = inputDir+"/"+img.inputPath;
+		const auto imgInInfo = QFileInfo(imgInPath);
+		if(!imgInInfo.exists())
+		{
+			qCritical() << "Failed to locate an image referenced in the description:" << img.inputPath;
+			continue;
+		}
+		const auto imgOutPath = outDir + "/" + img.outputPath;
+		const auto imgOutInfo = QFileInfo(imgOutPath);
+		bool targetExistsAndDiffers = imgOutInfo.exists() && imgInInfo.size() != imgOutInfo.size();
+		if(imgOutInfo.exists() && !targetExistsAndDiffers)
+		{
+			// Check that the contents are the same
+			QFile in(imgInPath);
+			if(!in.open(QFile::ReadOnly))
+			{
+				qCritical().noquote() << "Failed to open file" << imgInPath;
+				continue;
+			}
+			QFile out(imgOutPath);
+			if(!out.open(QFile::ReadOnly))
+			{
+				qCritical().noquote() << "Failed to open file" << imgOutPath;
+				continue;
+			}
+			targetExistsAndDiffers = in.readAll() != out.readAll();
+		}
+		if(targetExistsAndDiffers)
+		{
+			qCritical() << "Image file names collide:" << img.inputPath << "and" << img.outputPath;
+			continue;
+		}
+
+		if(imgOutInfo.exists())
 		{
-			qCritical() << "Failed to locate an image referenced in the description:" << img;
+			// It exists, but is the same as the file we're trying to copy, so skip it
 			continue;
 		}
-		const auto imgOutPath = outDir + "/" + img;
+
 		const auto imgDir = QFileInfo(imgOutPath).absoluteDir().absolutePath();
 		if(!QDir().mkpath(imgDir))
 		{
-			qCritical() << "Failed to create output directory for image file" << img;
+			qCritical() << "Failed to create output directory for image file" << img.outputPath;
 			continue;
 		}
 
 		if(!QFile(imgInPath).copy(imgOutPath))
 		{
-			qCritical() << "Failed to copy an image file referenced in the description:" << img;
+			qCritical() << "Failed to copy an image file referenced in the description:" << img.inputPath << "to" << img.outputPath;
 			continue;
 		}
 	}

diff --git a/util/skyculture-converter/DescriptionOldLoader.hpp b/util/skyculture-converter/DescriptionOldLoader.hpp
@@ -10,7 +10,15 @@ class DescriptionOldLoader
 	QString markdown;
 	QHash<QString/*locale*/, QString/*markdown*/> translatedMDs;
 	QString inputDir;
-	std::vector<QString> imageHRefs;
+	struct ImageHRef
+	{
+		ImageHRef(QString const& inputPath, QString const& outputPath)
+			: inputPath(inputPath), outputPath(outputPath)
+		{}
+		QString inputPath;
+		QString outputPath;
+	};
+	std::vector<ImageHRef> imageHRefs;
 	struct DictEntry
 	{
 		QString comment;
@@ -26,7 +34,7 @@ class DescriptionOldLoader
 	QHash<QString/*locale*/, TranslationDict> translations;
 	QHash<QString/*locale*/, QString/*header*/> poHeaders;
 	bool dumpMarkdown(const QString& outDir) const;
-	void locateAllInlineImages(const QString& html);
+	void locateAndRelocateAllInlineImages(QString& html, bool saveToRefs);
 	void loadTranslationsOfNames(const QString& poBaseDir, const QString& cultureId, const QString& englishName);
 	QString translateSection(const QString& markdown, const qsizetype bodyStartPos, const qsizetype bodyEndPos, const QString& locale, const QString& sectionName) const;
 	QString translateDescription(const QString& markdown, const QString& locale) const;