From 64811c44df5ce7697039e646ea5c0debeaafdc2b Mon Sep 17 00:00:00 2001 From: Aleksandr Tretyakov Date: Mon, 29 Aug 2022 23:43:13 +0500 Subject: [PATCH 01/12] Show an error if there is an undeclared type or template. --- Kodgen/Include/Kodgen/Parsing/FileParser.h | 10 +++ Kodgen/Source/Parsing/FileParser.cpp | 85 +++++++++++++++++----- 2 files changed, 75 insertions(+), 20 deletions(-) diff --git a/Kodgen/Include/Kodgen/Parsing/FileParser.h b/Kodgen/Include/Kodgen/Parsing/FileParser.h index 5b51bf9b..a9162540 100644 --- a/Kodgen/Include/Kodgen/Parsing/FileParser.h +++ b/Kodgen/Include/Kodgen/Parsing/FileParser.h @@ -109,6 +109,16 @@ namespace kodgen */ bool logDiagnostic(CXTranslationUnit const& translationUnit) const noexcept; + /** + * @brief Looks if there were critical errors after parsing of the provided translation unit. + * An error is considered critical if it might cause incorrect type information when using reflection. + * + * @param translationUnit Translation unit we want to get errors of. + * + * @return array of critical errors (if found). + */ + std::vector findCriticalErrors(CXTranslationUnit const& translationUnit) const noexcept; + /** * @brief Helper to get the ParsingResult contained in the context as a FileParsingResult. * diff --git a/Kodgen/Source/Parsing/FileParser.cpp b/Kodgen/Source/Parsing/FileParser.cpp index 915d9e7e..f4563188 100644 --- a/Kodgen/Source/Parsing/FileParser.cpp +++ b/Kodgen/Source/Parsing/FileParser.cpp @@ -59,31 +59,43 @@ bool FileParser::parse(fs::path const& toParseFile, FileParsingResult& out_resul if (translationUnit != nullptr) { - ParsingContext& context = pushContext(translationUnit, out_result); - - if (clang_visitChildren(context.rootCursor, &FileParser::parseNestedEntity, this) || !out_result.errors.empty()) + // Look if there were critical errors. + const auto criticalErrors = findCriticalErrors(translationUnit); + if (criticalErrors.empty()) { - //ERROR + ParsingContext& context = pushContext(translationUnit, out_result); + + if (clang_visitChildren(context.rootCursor, &FileParser::parseNestedEntity, this) || !out_result.errors.empty()) + { + //ERROR + } + else + { + //Refresh all outer entities contained in the final result + refreshOuterEntity(out_result); + + isSuccess = true; + } + + popContext(); + + //There should not have any context left once parsing has finished + assert(contextsStack.empty()); + + if (_settings->shouldLogDiagnostic) + { + logDiagnostic(translationUnit); + } + + clang_disposeTranslationUnit(translationUnit); } else { - //Refresh all outer entities contained in the final result - refreshOuterEntity(out_result); - - isSuccess = true; - } - - popContext(); - - //There should not have any context left once parsing has finished - assert(contextsStack.empty()); - - if (_settings->shouldLogDiagnostic) - { - logDiagnostic(translationUnit); + for (const auto& message : criticalErrors) + { + out_result.errors.emplace_back(message); + } } - - clang_disposeTranslationUnit(translationUnit); } else { @@ -268,6 +280,39 @@ void FileParser::postParse(fs::path const&, FileParsingResult const&) noexcept */ } +std::vector FileParser::findCriticalErrors(CXTranslationUnit const& translationUnit) const noexcept +{ + const CXDiagnosticSet diagnostics = clang_getDiagnosticSetFromTU(translationUnit); + const unsigned int diagnosticsCount = clang_getNumDiagnosticsInSet(diagnostics); + + std::vector criticalErrors; + + for (unsigned i = 0u; i < diagnosticsCount; i++) + { + const CXDiagnostic diagnostic(clang_getDiagnosticInSet(diagnostics, i)); + + auto diagnosticMessage = Helpers::getString(clang_formatDiagnostic(diagnostic, clang_defaultDiagnosticDisplayOptions())); + + if (diagnosticMessage.find("error:") != std::string::npos) + { + if (diagnosticMessage.find("use of undeclared identifier") != std::string::npos) + { + criticalErrors.push_back(diagnosticMessage + " (did you forgot to include the header?)"); + } + else if (diagnosticMessage.find("no template named") != std::string::npos) + { + criticalErrors.push_back(diagnosticMessage + " (did you forgot to include the header?)"); + } + } + + clang_disposeDiagnostic(diagnostic); + } + + clang_disposeDiagnosticSet(diagnostics); + + return criticalErrors; +} + bool FileParser::logDiagnostic(CXTranslationUnit const& translationUnit) const noexcept { if (logger != nullptr) From fa8c9d24f11b78bf4ec854b9f4a81bacda4a6330 Mon Sep 17 00:00:00 2001 From: Aleksandr Tretyakov Date: Wed, 31 Aug 2022 20:23:59 +0500 Subject: [PATCH 02/12] Return accidentally removed function. --- .../Include/Kodgen/InfoStructures/TypeInfo.h | 12 +++++++ Kodgen/Source/InfoStructures/TypeInfo.cpp | 34 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/Kodgen/Include/Kodgen/InfoStructures/TypeInfo.h b/Kodgen/Include/Kodgen/InfoStructures/TypeInfo.h index e1e9ffbb..cf1ff2fc 100644 --- a/Kodgen/Include/Kodgen/InfoStructures/TypeInfo.h +++ b/Kodgen/Include/Kodgen/InfoStructures/TypeInfo.h @@ -175,6 +175,18 @@ namespace kodgen */ std::vector const& getTemplateParameters() const noexcept; + /** + * @brief Compute the template parameters signature if the type info is a templated type. + * Ex: template typename V> + * will output + * typename, int, template typename + * + * @param useAutoForNonTypeParams Should all non-type parameters be replaced by the auto keyword? + * + * @return The template signature of the type if the type info is a template type, else an empty string. + */ + std::string computeTemplateSignature(bool useAutoForNonTypeParams) const noexcept; + /** * @brief Check whether this type is template or not. * diff --git a/Kodgen/Source/InfoStructures/TypeInfo.cpp b/Kodgen/Source/InfoStructures/TypeInfo.cpp index 301bcfbc..1f50e427 100644 --- a/Kodgen/Source/InfoStructures/TypeInfo.cpp +++ b/Kodgen/Source/InfoStructures/TypeInfo.cpp @@ -389,6 +389,40 @@ std::vector const& TypeInfo::getTemplateParameters() const no return _templateParameters; } +std::string TypeInfo::computeTemplateSignature(bool useAutoForNonTypeParams) const noexcept +{ + std::string result; + + for (TemplateParamInfo const& templateParam : _templateParameters) + { + switch (templateParam.kind) + { + case ETemplateParameterKind::TypeTemplateParameter: + result += "typename"; + break; + + case ETemplateParameterKind::NonTypeTemplateParameter: + result += useAutoForNonTypeParams ? "auto" : templateParam.type->getName(); + break; + + case ETemplateParameterKind::TemplateTemplateParameter: + result += "template <" + templateParam.type->computeTemplateSignature(useAutoForNonTypeParams) + "> typename"; + break; + + default: + result += "RFK_UNDEFINED"; + break; + } + + result += ","; + } + + //Remove last "," + result.pop_back(); + + return result; +} + bool TypeInfo::isTemplateType() const noexcept { return !_templateParameters.empty(); From 2cfc318940218919b21ca0a84126ad3df16de62b Mon Sep 17 00:00:00 2001 From: Aleksandr Tretyakov Date: Wed, 31 Aug 2022 20:57:36 +0500 Subject: [PATCH 03/12] Only check for std::string and std::vector types for now. --- Kodgen/Include/Kodgen/Parsing/FileParser.h | 10 +++++----- Kodgen/Source/Parsing/FileParser.cpp | 20 +++++++++++--------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/Kodgen/Include/Kodgen/Parsing/FileParser.h b/Kodgen/Include/Kodgen/Parsing/FileParser.h index a9162540..1328411b 100644 --- a/Kodgen/Include/Kodgen/Parsing/FileParser.h +++ b/Kodgen/Include/Kodgen/Parsing/FileParser.h @@ -110,14 +110,14 @@ namespace kodgen bool logDiagnostic(CXTranslationUnit const& translationUnit) const noexcept; /** - * @brief Looks if there were critical errors after parsing of the provided translation unit. - * An error is considered critical if it might cause incorrect type information when using reflection. + * @brief Looks if there were some errors related to not found STL types after parsing of the + * provided translation unit (which might cause incorrect type information when using reflection). * - * @param translationUnit Translation unit we want to get errors of. + * @param translationUnit Translation unit we want to check. * - * @return array of critical errors (if found). + * @return array of errors (if found). */ - std::vector findCriticalErrors(CXTranslationUnit const& translationUnit) const noexcept; + std::vector findStandardTypesErrors(CXTranslationUnit const& translationUnit) const noexcept; /** * @brief Helper to get the ParsingResult contained in the context as a FileParsingResult. diff --git a/Kodgen/Source/Parsing/FileParser.cpp b/Kodgen/Source/Parsing/FileParser.cpp index f4563188..028d6d4d 100644 --- a/Kodgen/Source/Parsing/FileParser.cpp +++ b/Kodgen/Source/Parsing/FileParser.cpp @@ -59,8 +59,8 @@ bool FileParser::parse(fs::path const& toParseFile, FileParsingResult& out_resul if (translationUnit != nullptr) { - // Look if there were critical errors. - const auto criticalErrors = findCriticalErrors(translationUnit); + // Look if there were errors. + const auto criticalErrors = findStandardTypesErrors(translationUnit); if (criticalErrors.empty()) { ParsingContext& context = pushContext(translationUnit, out_result); @@ -280,7 +280,7 @@ void FileParser::postParse(fs::path const&, FileParsingResult const&) noexcept */ } -std::vector FileParser::findCriticalErrors(CXTranslationUnit const& translationUnit) const noexcept +std::vector FileParser::findStandardTypesErrors(CXTranslationUnit const& translationUnit) const noexcept { const CXDiagnosticSet diagnostics = clang_getDiagnosticSetFromTU(translationUnit); const unsigned int diagnosticsCount = clang_getNumDiagnosticsInSet(diagnostics); @@ -295,13 +295,15 @@ std::vector FileParser::findCriticalErrors(CXTranslationUnit const& if (diagnosticMessage.find("error:") != std::string::npos) { - if (diagnosticMessage.find("use of undeclared identifier") != std::string::npos) + // Check for std::string and std::vector types. + if (diagnosticMessage.find("use of undeclared identifier 'std'") != std::string::npos || + diagnosticMessage.find("use of undeclared identifier 'string'") != std::string::npos || + diagnosticMessage.find("unknown type name 'string'") != std::string::npos || + diagnosticMessage.find("no member named 'string' in namespace 'std'") != std::string::npos || + diagnosticMessage.find("no template named 'vector' in namespace 'std'") != std::string::npos || + diagnosticMessage.find("no template named 'vector'") != std::string::npos) { - criticalErrors.push_back(diagnosticMessage + " (did you forgot to include the header?)"); - } - else if (diagnosticMessage.find("no template named") != std::string::npos) - { - criticalErrors.push_back(diagnosticMessage + " (did you forgot to include the header?)"); + criticalErrors.push_back(diagnosticMessage + " (please, include the header)"); } } From 06e07a12ed92104b993a63bc218430b2ac257d97 Mon Sep 17 00:00:00 2001 From: Aleksandr Tretyakov Date: Wed, 31 Aug 2022 21:54:59 +0500 Subject: [PATCH 04/12] Add location information to error messages. --- Kodgen/Source/Parsing/FileParser.cpp | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/Kodgen/Source/Parsing/FileParser.cpp b/Kodgen/Source/Parsing/FileParser.cpp index 028d6d4d..bff1b7b6 100644 --- a/Kodgen/Source/Parsing/FileParser.cpp +++ b/Kodgen/Source/Parsing/FileParser.cpp @@ -60,8 +60,8 @@ bool FileParser::parse(fs::path const& toParseFile, FileParsingResult& out_resul if (translationUnit != nullptr) { // Look if there were errors. - const auto criticalErrors = findStandardTypesErrors(translationUnit); - if (criticalErrors.empty()) + const auto typeErrors = findStandardTypesErrors(translationUnit); + if (typeErrors.empty()) { ParsingContext& context = pushContext(translationUnit, out_result); @@ -91,7 +91,7 @@ bool FileParser::parse(fs::path const& toParseFile, FileParsingResult& out_resul } else { - for (const auto& message : criticalErrors) + for (const auto& message : typeErrors) { out_result.errors.emplace_back(message); } @@ -303,7 +303,17 @@ std::vector FileParser::findStandardTypesErrors(CXTranslationUnit c diagnosticMessage.find("no template named 'vector' in namespace 'std'") != std::string::npos || diagnosticMessage.find("no template named 'vector'") != std::string::npos) { - criticalErrors.push_back(diagnosticMessage + " (please, include the header)"); + CXFile file; + unsigned line, column; + clang_getExpansionLocation( + clang_getDiagnosticLocation(diagnostic), + &file, + &line, + &column, + nullptr); + auto location = Helpers::getString(clang_getFileName(file)); + location += ", line " + std::to_string(line) + ", column " + std::to_string(column) + ""; + criticalErrors.push_back(diagnosticMessage + " (please, include the header) (" + location + ")"); } } From 9bf42483daeb5a11c89427c6e1ee712322278e9d Mon Sep 17 00:00:00 2001 From: Aleksandr Tretyakov Date: Thu, 1 Sep 2022 19:02:32 +0500 Subject: [PATCH 05/12] Fail on any libclang error, properly handle GENERATED macros errors. --- .../Include/Kodgen/CodeGen/CodeGenManager.inl | 6 +- Kodgen/Include/Kodgen/Parsing/FileParser.h | 54 ++++-- Kodgen/Source/Parsing/FileParser.cpp | 168 ++++++++++++++---- 3 files changed, 183 insertions(+), 45 deletions(-) diff --git a/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl b/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl index 589445fa..82230977 100644 --- a/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl +++ b/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl @@ -16,6 +16,8 @@ void CodeGenManager::processFiles(FileParserType& fileParser, CodeGenUnitType& c //Launch all parsing -> generation processes std::shared_ptr parsingTask; + + const kodgen::MacroCodeGenUnitSettings* codeGenSettings = codeGenUnit.getSettings(); for (int i = 0; i < iterationCount; i++) { @@ -24,13 +26,13 @@ void CodeGenManager::processFiles(FileParserType& fileParser, CodeGenUnitType& c for (fs::path const& file : toProcessFiles) { - auto parsingTaskLambda = [&fileParser, &file](TaskBase*) -> FileParsingResult + auto parsingTaskLambda = [codeGenSettings, &fileParser, &file](TaskBase*) -> FileParsingResult { //Copy a parser for this task FileParserType fileParserCopy = fileParser; FileParsingResult parsingResult; - fileParserCopy.parse(file, parsingResult); + fileParserCopy.parse(file, parsingResult, codeGenSettings); return parsingResult; }; diff --git a/Kodgen/Include/Kodgen/Parsing/FileParser.h b/Kodgen/Include/Kodgen/Parsing/FileParser.h index 1328411b..ecc8186b 100644 --- a/Kodgen/Include/Kodgen/Parsing/FileParser.h +++ b/Kodgen/Include/Kodgen/Parsing/FileParser.h @@ -9,6 +9,7 @@ #include #include +#include #include //std::shared_ptr #include @@ -19,6 +20,7 @@ #include "Kodgen/Parsing/PropertyParser.h" #include "Kodgen/Misc/Filesystem.h" #include "Kodgen/Misc/ILogger.h" +#include namespace kodgen { @@ -110,14 +112,44 @@ namespace kodgen bool logDiagnostic(CXTranslationUnit const& translationUnit) const noexcept; /** - * @brief Looks if there were some errors related to not found STL types after parsing of the - * provided translation unit (which might cause incorrect type information when using reflection). + * Splits class footer macro pattern into two parts between "##...##" substring. For example: + * for "##CLASSFULLNAME##_GENERATED" it will return a pair of "" (empty) and "_GENERATED". + * + * @param classFooterMacroPattern class footer macro pattern. + * + * @return a pair of strings between "##...##" substring. + */ + static std::pair splitClassFooterMacroPattern(const std::string& classFooterMacroPattern); + + /** + * Clears the file and adds "#define" statements for the specified macros. + * + * @param filePath File in which to define macros. + * @param macroNamesToDefine Macro names to define. + * + * @retun true if successfull, false is failed. + */ + static bool populateFileWithMacros(const fs::path& filePath, const std::set& macroNamesToDefine); + + /** + * @brief Returns all errors found during translation unit parsing. + * + * @parma toParseFile File that was used in parsing. + * @param translationUnit Translation unit we want to check. + * @param codeGenSettings Code generation settings that was used. + * @param notFoundGeneratedMacroNames Set of GENERATED macro names that were not found + * during parsing. + * + * @remark This function ignores errors caused by including generated headers or + * using GENERATED or reflection macros. * - * @param translationUnit Translation unit we want to check. - * * @return array of errors (if found). */ - std::vector findStandardTypesErrors(CXTranslationUnit const& translationUnit) const noexcept; + std::vector getErrors( + fs::path const& toParseFile, + CXTranslationUnit const& translationUnit, + const kodgen::MacroCodeGenUnitSettings* codeGenSettings, + std::set& notFoundGeneratedMacroNames) const noexcept; /** * @brief Helper to get the ParsingResult contained in the context as a FileParsingResult. @@ -130,9 +162,9 @@ namespace kodgen /** * @brief Overridable method called just before starting the parsing process of a file * - * @param parseFile Path to the file which is about to be parsed + * @param parseFile Path to the file which is about to be parsed. */ - virtual void preParse(fs::path const& parseFile) noexcept; + virtual void preParse(fs::path const& parseFile) noexcept; /** * @brief Overridable method called just after the parsing process has been finished @@ -155,13 +187,15 @@ namespace kodgen /** * @brief Parse the file and fill the FileParsingResult. * - * @param toParseFile Path to the file to parse. - * @param out_result Result filled while parsing the file. + * @param toParseFile Path to the file to parse. + * @param out_result Result filled while parsing the file. + * @param codeGenSettings Code generation settings. * * @return true if the parsing process finished without error, else false */ bool parse(fs::path const& toParseFile, - FileParsingResult& out_result) noexcept; + FileParsingResult& out_result, + const kodgen::MacroCodeGenUnitSettings* codeGenSettings) noexcept; /** * @brief Getter for _settings field. diff --git a/Kodgen/Source/Parsing/FileParser.cpp b/Kodgen/Source/Parsing/FileParser.cpp index bff1b7b6..0d9150e9 100644 --- a/Kodgen/Source/Parsing/FileParser.cpp +++ b/Kodgen/Source/Parsing/FileParser.cpp @@ -1,7 +1,6 @@ #include "Kodgen/Parsing/FileParser.h" #include - #include "Kodgen/Misc/Helpers.h" #include "Kodgen/Misc/DisableWarningMacros.h" #include "Kodgen/Misc/TomlUtility.h" @@ -41,7 +40,25 @@ FileParser::~FileParser() noexcept } } -bool FileParser::parse(fs::path const& toParseFile, FileParsingResult& out_result) noexcept +bool FileParser::populateFileWithMacros(const fs::path& filePath, const std::set& macroNamesToDefine) +{ + std::ofstream generatedfile(filePath, std::ios::app); + if (!generatedfile.is_open()) + { + return false; + } + + for (const auto& macroName : macroNamesToDefine) + { + generatedfile << "#define " + macroName + " " << std::endl; + } + + generatedfile.close(); + + return true; +} + +bool FileParser::parse(fs::path const& toParseFile, FileParsingResult& out_result, const kodgen::MacroCodeGenUnitSettings* codeGenSettings) noexcept { assert(_settings.use_count() != 0); @@ -54,14 +71,51 @@ bool FileParser::parse(fs::path const& toParseFile, FileParsingResult& out_resul //Fill the parsed file info out_result.parsedFile = FilesystemHelpers::sanitizePath(toParseFile); - //Parse the given file - CXTranslationUnit translationUnit = clang_parseTranslationUnit(_clangIndex, toParseFile.string().c_str(), _settings->getCompilationArguments().data(), static_cast(_settings->getCompilationArguments().size()), nullptr, 0, CXTranslationUnit_SkipFunctionBodies | CXTranslationUnit_Incomplete | CXTranslationUnit_KeepGoing); + // Prepare arrays for errors. + std::set notFoundGeneratedMacroNames; + std::vector errors; + + // Prepare variables for parsing. + bool rerunParsing = false; + bool populatedGeneratedFileWithMacros = false; + CXTranslationUnit translationUnit; + do + { + rerunParsing = false; + + // Parse the given file. + translationUnit = clang_parseTranslationUnit(_clangIndex, toParseFile.string().c_str(), _settings->getCompilationArguments().data(), static_cast(_settings->getCompilationArguments().size()), nullptr, 0, CXTranslationUnit_SkipFunctionBodies | CXTranslationUnit_Incomplete | CXTranslationUnit_KeepGoing); + if (translationUnit != nullptr) + { + errors = getErrors(toParseFile, translationUnit, codeGenSettings, notFoundGeneratedMacroNames); + if (!populatedGeneratedFileWithMacros && !notFoundGeneratedMacroNames.empty()) + { + const auto generatedFilePath = codeGenSettings->getOutputDirectory() / codeGenSettings->getGeneratedHeaderFileName(toParseFile); + if (!populateFileWithMacros(generatedFilePath, notFoundGeneratedMacroNames)) + { + out_result.errors.emplace_back("Failed to populate the generated file " + generatedFilePath.string() + " with macros."); + postParse(toParseFile, out_result); + return false; + } + + notFoundGeneratedMacroNames.clear(); + populatedGeneratedFileWithMacros = true; + rerunParsing = true; + } + else if (populatedGeneratedFileWithMacros) + { + // Clear generated file as it will be filled with an actual information. + const auto generatedFilePath = codeGenSettings->getOutputDirectory() / codeGenSettings->getGeneratedHeaderFileName(toParseFile); + std::ofstream file(generatedFilePath); // truncate the file + file.close(); + } + } + }while(rerunParsing); + if (translationUnit != nullptr) { - // Look if there were errors. - const auto typeErrors = findStandardTypesErrors(translationUnit); - if (typeErrors.empty()) + if (errors.empty()) { ParsingContext& context = pushContext(translationUnit, out_result); @@ -91,7 +145,7 @@ bool FileParser::parse(fs::path const& toParseFile, FileParsingResult& out_resul } else { - for (const auto& message : typeErrors) + for (const auto& message : errors) { out_result.errors.emplace_back(message); } @@ -280,49 +334,97 @@ void FileParser::postParse(fs::path const&, FileParsingResult const&) noexcept */ } -std::vector FileParser::findStandardTypesErrors(CXTranslationUnit const& translationUnit) const noexcept +std::pair FileParser::splitClassFooterMacroPattern(const std::string& classFooterMacroPattern) +{ + // Get left text. + const auto leftSharpPos = classFooterMacroPattern.find('#'); + if (leftSharpPos == std::string::npos) + { + return std::make_pair("", ""); + } + std::string leftText; + if (leftSharpPos != 0) + { + leftText = classFooterMacroPattern.substr(0, leftSharpPos); + } + + // Get right text. + const auto rightSharpPos = classFooterMacroPattern.rfind('#'); + if (rightSharpPos == std::string::npos) + { + return std::make_pair("", ""); + } + std::string rightText; + if (rightSharpPos != classFooterMacroPattern.size()) + { + rightText = classFooterMacroPattern.substr(rightSharpPos + 1); + } + + return std::make_pair(leftText, rightText); +} + +std::vector FileParser::getErrors( + fs::path const& toParseFile, + CXTranslationUnit const& translationUnit, + const kodgen::MacroCodeGenUnitSettings* codeGenSettings, + std::set& notFoundGeneratedMacroNames) const noexcept { const CXDiagnosticSet diagnostics = clang_getDiagnosticSetFromTU(translationUnit); const unsigned int diagnosticsCount = clang_getNumDiagnosticsInSet(diagnostics); - - std::vector criticalErrors; + const std::string generatedHeaderFilename = codeGenSettings->getGeneratedHeaderFileName(toParseFile).filename().string(); + const std::string fileGeneratedMacroName = codeGenSettings->getHeaderFileFooterMacro(toParseFile); + const auto [leftClassFooterMacroText, rightClassFooterMacroText] = splitClassFooterMacroPattern(codeGenSettings->getClassFooterMacroPattern()); + if (leftClassFooterMacroText.empty() && rightClassFooterMacroText.empty()) + { + return {"failed to split class footer macro pattern"}; + } + const std::string unknownTypeNameError = "unknown type name '"; + + std::vector errors; for (unsigned i = 0u; i < diagnosticsCount; i++) { const CXDiagnostic diagnostic(clang_getDiagnosticInSet(diagnostics, i)); + //auto diagnosticMessage = Helpers::getString(clang_formatDiagnostic(diagnostic, clang_defaultDiagnosticDisplayOptions())); + auto diagnosticMessage = Helpers::getString(clang_getDiagnosticSpelling(diagnostic)); - auto diagnosticMessage = Helpers::getString(clang_formatDiagnostic(diagnostic, clang_defaultDiagnosticDisplayOptions())); - - if (diagnosticMessage.find("error:") != std::string::npos) + // Look if this error is related to GENERATED macros like "unknown type name 'File_MyClass_GENERATED'". + if (diagnosticMessage.find(unknownTypeNameError) != std::string::npos) { - // Check for std::string and std::vector types. - if (diagnosticMessage.find("use of undeclared identifier 'std'") != std::string::npos || - diagnosticMessage.find("use of undeclared identifier 'string'") != std::string::npos || - diagnosticMessage.find("unknown type name 'string'") != std::string::npos || - diagnosticMessage.find("no member named 'string' in namespace 'std'") != std::string::npos || - diagnosticMessage.find("no template named 'vector' in namespace 'std'") != std::string::npos || - diagnosticMessage.find("no template named 'vector'") != std::string::npos) + const auto unknownTypeName = diagnosticMessage.substr( // don't include last ' character + unknownTypeNameError.size(), diagnosticMessage.size() - 1 - unknownTypeNameError.size()); + if (unknownTypeName == fileGeneratedMacroName) { - CXFile file; - unsigned line, column; - clang_getExpansionLocation( - clang_getDiagnosticLocation(diagnostic), - &file, - &line, - &column, - nullptr); - auto location = Helpers::getString(clang_getFileName(file)); - location += ", line " + std::to_string(line) + ", column " + std::to_string(column) + ""; - criticalErrors.push_back(diagnosticMessage + " (please, include the header) (" + location + ")"); + notFoundGeneratedMacroNames.insert(fileGeneratedMacroName); + continue; + } + else if (unknownTypeName.find(leftClassFooterMacroText) != std::string::npos && + unknownTypeName.find(rightClassFooterMacroText) != std::string::npos) + { + notFoundGeneratedMacroNames.insert(unknownTypeName); + continue; } } + + // Prepare location information. + CXFile file; + unsigned line, column; + clang_getExpansionLocation( + clang_getDiagnosticLocation(diagnostic), + &file, + &line, + &column, + nullptr); + auto location = file ? Helpers::getString(clang_getFileName(file)) : std::string(); + location += ", line " + std::to_string(line) + ", column " + std::to_string(column) + ""; + errors.push_back(diagnosticMessage + " (" + location + ")"); clang_disposeDiagnostic(diagnostic); } clang_disposeDiagnosticSet(diagnostics); - return criticalErrors; + return errors; } bool FileParser::logDiagnostic(CXTranslationUnit const& translationUnit) const noexcept From dfc2186cb4de94346a0322539c7403e6b8100248 Mon Sep 17 00:00:00 2001 From: Aleksandr Tretyakov Date: Thu, 1 Sep 2022 21:26:26 +0500 Subject: [PATCH 06/12] Add a new parameter 'additionalClangArguments' to configuration. --- Kodgen/Include/Kodgen/Parsing/ParsingSettings.h | 11 +++++++++++ Kodgen/KodgenSettings.toml | 3 +++ Kodgen/Source/Parsing/ParsingSettings.cpp | 11 +++++++++++ 3 files changed, 25 insertions(+) diff --git a/Kodgen/Include/Kodgen/Parsing/ParsingSettings.h b/Kodgen/Include/Kodgen/Parsing/ParsingSettings.h index 39511680..85212ecb 100644 --- a/Kodgen/Include/Kodgen/Parsing/ParsingSettings.h +++ b/Kodgen/Include/Kodgen/Parsing/ParsingSettings.h @@ -38,6 +38,8 @@ namespace kodgen */ std::string _compilerExeName = ""; + std::string _additionalClangArguments; + /** Variables used to build compilation command line. */ std::string _kodgenParsingMacro = "-D" + parsingMacro; std::string _cppVersionCommandLine; @@ -123,6 +125,15 @@ namespace kodgen void loadCompilerExeName(toml::value const& parsingSettings, ILogger* logger) noexcept; + /** + * @brief Load the additionalClangArguments setting from toml. + * + * @param parsingSettings Toml content. + * @param logger Optional logger used to issue loading logs. Can be nullptr. + */ + void loadAdditionalClangArguments(toml::value const& parsingSettings, + ILogger* logger) noexcept; + /** * @brief Load the _projectIncludeDirectories setting from toml. * Loaded directories completely replace previous _projectIncludeDirectories if any. diff --git a/Kodgen/KodgenSettings.toml b/Kodgen/KodgenSettings.toml index cddb0e7c..97bfed10 100644 --- a/Kodgen/KodgenSettings.toml +++ b/Kodgen/KodgenSettings.toml @@ -40,6 +40,9 @@ projectIncludeDirectories = [ # Must be one of "msvc", "clang++", "g++" compilerExeName = "clang++" +# Additional arguments to add to libclang +additionalClangArguments = "" + # Abort parsing on first encountered error shouldAbortParsingOnFirstError = true diff --git a/Kodgen/Source/Parsing/ParsingSettings.cpp b/Kodgen/Source/Parsing/ParsingSettings.cpp index d1a20b7c..8ec70821 100644 --- a/Kodgen/Source/Parsing/ParsingSettings.cpp +++ b/Kodgen/Source/Parsing/ParsingSettings.cpp @@ -104,6 +104,8 @@ void ParsingSettings::refreshCompilationArguments(ILogger* logger) noexcept //Parsing C++ _compilationArguments.emplace_back("-xc++"); + _compilationArguments.emplace_back(_additionalClangArguments.data()); + #if KODGEN_DEV _compilationArguments.emplace_back("-v"); #endif @@ -142,6 +144,7 @@ bool ParsingSettings::loadSettingsValues(toml::value const& tomlData, ILogger* l loadShouldLogDiagnostic(tomlParsingSettings, logger); loadCompilerExeName(tomlParsingSettings, logger); loadProjectIncludeDirectories(tomlParsingSettings, logger); + loadAdditionalClangArguments(tomlParsingSettings, logger); return propertyParsingSettings.loadSettingsValues(tomlParsingSettings, logger); } @@ -234,6 +237,14 @@ void ParsingSettings::loadShouldAbortParsingOnFirstError(toml::value const& toml } } +void ParsingSettings::loadAdditionalClangArguments(toml::value const& tomlFileParsingSettings, ILogger* logger) noexcept +{ + if (TomlUtility::updateSetting(tomlFileParsingSettings, "additionalClangArguments", _additionalClangArguments, logger) && logger != nullptr) + { + logger->log("[TOML] Load additionalClangArguments: " + _additionalClangArguments); + } +} + void ParsingSettings::loadShouldLogDiagnostic(toml::value const& tomlFileParsingSettings, ILogger* logger) noexcept { if (TomlUtility::updateSetting(tomlFileParsingSettings, "shouldLogDiagnostic", shouldLogDiagnostic, logger) && logger != nullptr) From 778a02f077fe6a190660dc706dea98ae5b8234e6 Mon Sep 17 00:00:00 2001 From: Aleksandr Tretyakov Date: Fri, 2 Sep 2022 22:39:50 +0500 Subject: [PATCH 07/12] Split code gen into 3 steps: pre-parsing, parsing and generation. --- .../Include/Kodgen/CodeGen/CodeGenManager.inl | 71 +++++++++--- Kodgen/Include/Kodgen/Parsing/FileParser.h | 21 +++- Kodgen/Source/Parsing/FileParser.cpp | 108 +++++++++--------- 3 files changed, 131 insertions(+), 69 deletions(-) diff --git a/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl b/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl index 82230977..062c173a 100644 --- a/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl +++ b/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl @@ -9,6 +9,7 @@ template void CodeGenManager::processFiles(FileParserType& fileParser, CodeGenUnitType& codeGenUnit, std::set const& toProcessFiles, CodeGenResult& out_genResult) noexcept { std::vector> generationTasks; + std::vector> parsingTasks; uint8 iterationCount = codeGenUnit.getIterationCount(); //Reserve enough space for all tasks @@ -16,6 +17,7 @@ void CodeGenManager::processFiles(FileParserType& fileParser, CodeGenUnitType& c //Launch all parsing -> generation processes std::shared_ptr parsingTask; + std::shared_ptr preParsingTask; const kodgen::MacroCodeGenUnitSettings* codeGenSettings = codeGenUnit.getSettings(); @@ -24,19 +26,69 @@ void CodeGenManager::processFiles(FileParserType& fileParser, CodeGenUnitType& c //Lock the thread pool until all tasks have been pushed to avoid competing for the tasks mutex _threadPool.setIsRunning(false); + // First, run pre-parse step in which we will fill generated files + // with reflection macros (file/class macros). + // This will avoid the following issue: if we are using inheritance and we haven't + // generated parent's data while parsing child class we will fail with an error. + for (fs::path const& file : toProcessFiles) + { + auto preParsingTaskLambda = [codeGenSettings, &fileParser, &file](TaskBase*) -> bool + { + FileParserType fileParserCopy = fileParser; + return fileParserCopy.prepareForParsing(file, codeGenSettings); + }; + + preParsingTask = _threadPool.submitTask(std::string("Pre-parsing ") + std::to_string(i), preParsingTaskLambda); + } + + // Wait for pre-parse step to finish. + _threadPool.setIsRunning(true); + _threadPool.joinWorkers(); + _threadPool.setIsRunning(false); + + // Run parsing step. for (fs::path const& file : toProcessFiles) { auto parsingTaskLambda = [codeGenSettings, &fileParser, &file](TaskBase*) -> FileParsingResult { + FileParsingResult parsingResult; + //Copy a parser for this task FileParserType fileParserCopy = fileParser; - FileParsingResult parsingResult; fileParserCopy.parse(file, parsingResult, codeGenSettings); return parsingResult; }; + //Add file to the list of parsed files before starting the task to avoid having to synchronize threads + out_genResult.parsedFiles.push_back(file); + + //Parse files + //For multiple iterations on a same file, the parsing task depends on the previous generation task for the same file + parsingTask = _threadPool.submitTask(std::string("Parsing ") + std::to_string(i), parsingTaskLambda); + parsingTasks.push_back(parsingTask); + } + + // Wait for parse step to finish. + // We wait here for all tasks to be finished because on the next step + // (the generation step) we will fill generated files with an actual data + // and this will clear existing macro defines that generated files have, + // but we need these macros for all parsing tasks to be finished without errors. + _threadPool.setIsRunning(true); + _threadPool.joinWorkers(); + _threadPool.setIsRunning(false); + + // Run code generation after all files were parsed. + size_t parsingTaskIndex = 0; + for (fs::path const& file : toProcessFiles) + { + // Clear generated file as it will be filled with an actual information. + // Right now it has some defines that were used for proper parsing. + const auto generatedFilePath = codeGenSettings->getOutputDirectory() / codeGenSettings->getGeneratedHeaderFileName(file); + std::ofstream generatedFile(generatedFilePath); // truncate the file + generatedFile.close(); + auto generationTaskLambda = [&codeGenUnit](TaskBase* parsingTask) -> CodeGenResult { CodeGenResult out_generationResult; @@ -55,22 +107,15 @@ void CodeGenManager::processFiles(FileParserType& fileParser, CodeGenUnitType& c return out_generationResult; }; - - //Add file to the list of parsed files before starting the task to avoid having to synchronize threads - out_genResult.parsedFiles.push_back(file); - - //Parse files - //For multiple iterations on a same file, the parsing task depends on the previous generation task for the same file - parsingTask = _threadPool.submitTask(std::string("Parsing ") + std::to_string(i), parsingTaskLambda); - - //Generate code - generationTasks.emplace_back(_threadPool.submitTask(std::string("Generation ") + std::to_string(i), generationTaskLambda, { parsingTask })); + + generationTasks.emplace_back(_threadPool.submitTask(std::string("Generation ") + std::to_string(i), generationTaskLambda, { parsingTasks[parsingTaskIndex] })); + parsingTaskIndex += 1; } - //Wait for this iteration to complete before continuing any further - //(an iteration N depends on the iteration N - 1) + // Wait for code generation. _threadPool.setIsRunning(true); _threadPool.joinWorkers(); + parsingTasks.clear(); } //Merge all generation results together diff --git a/Kodgen/Include/Kodgen/Parsing/FileParser.h b/Kodgen/Include/Kodgen/Parsing/FileParser.h index ecc8186b..b14fb6f8 100644 --- a/Kodgen/Include/Kodgen/Parsing/FileParser.h +++ b/Kodgen/Include/Kodgen/Parsing/FileParser.h @@ -112,14 +112,14 @@ namespace kodgen bool logDiagnostic(CXTranslationUnit const& translationUnit) const noexcept; /** - * Splits class footer macro pattern into two parts between "##...##" substring. For example: + * Splits macro pattern into two parts between "##...##" substring. For example: * for "##CLASSFULLNAME##_GENERATED" it will return a pair of "" (empty) and "_GENERATED". * - * @param classFooterMacroPattern class footer macro pattern. + * @param macroPattern Macro pattern. * * @return a pair of strings between "##...##" substring. */ - static std::pair splitClassFooterMacroPattern(const std::string& classFooterMacroPattern); + static std::pair splitMacroPattern(const std::string& macroPattern); /** * Clears the file and adds "#define" statements for the specified macros. @@ -184,6 +184,17 @@ namespace kodgen FileParser(FileParser&&) noexcept; virtual ~FileParser() noexcept; + /** + * @brief Prepares initial generated file for parsing. + * + * @param toParseFile Path to the file to parse. + * @param codeGenSettings Code generation settings. + * + * @return true if the parsing process finished without error, else false + */ + bool prepareForParsing(fs::path const& toParseFile, + const kodgen::MacroCodeGenUnitSettings* codeGenSettings) noexcept; + /** * @brief Parse the file and fill the FileParsingResult. * @@ -193,8 +204,8 @@ namespace kodgen * * @return true if the parsing process finished without error, else false */ - bool parse(fs::path const& toParseFile, - FileParsingResult& out_result, + bool parse(fs::path const& toParseFile, + FileParsingResult& out_result, const kodgen::MacroCodeGenUnitSettings* codeGenSettings) noexcept; /** diff --git a/Kodgen/Source/Parsing/FileParser.cpp b/Kodgen/Source/Parsing/FileParser.cpp index 0d9150e9..ae8cc979 100644 --- a/Kodgen/Source/Parsing/FileParser.cpp +++ b/Kodgen/Source/Parsing/FileParser.cpp @@ -58,6 +58,44 @@ bool FileParser::populateFileWithMacros(const fs::path& filePath, const std::set return true; } +bool FileParser::prepareForParsing(fs::path const& toParseFile, const kodgen::MacroCodeGenUnitSettings* codeGenSettings) noexcept +{ + assert(_settings.use_count() != 0); + + logger->log("Starting pre-parsing step for " + toParseFile.string(), ILogger::ELogSeverity::Info); + + if (!fs::exists(toParseFile) || fs::is_directory(toParseFile)) return false; + + // Do initial parsing. + CXTranslationUnit translationUnit = clang_parseTranslationUnit(_clangIndex, toParseFile.string().c_str(), _settings->getCompilationArguments().data(), static_cast(_settings->getCompilationArguments().size()), nullptr, 0, CXTranslationUnit_SkipFunctionBodies | CXTranslationUnit_Incomplete | CXTranslationUnit_KeepGoing); + if (!translationUnit) + { + logger->log("Failed to initialize translation unit for file: " + toParseFile.string(), ILogger::ELogSeverity::Error); + return false; + } + + // Process errors. + std::set notFoundGeneratedMacroNames; + const auto errors = getErrors(toParseFile, translationUnit, codeGenSettings, notFoundGeneratedMacroNames); + + if (!notFoundGeneratedMacroNames.empty()) + { + // Populate generated file with macros. + const auto generatedFilePath = codeGenSettings->getOutputDirectory() / codeGenSettings->getGeneratedHeaderFileName(toParseFile); + if (!populateFileWithMacros(generatedFilePath, notFoundGeneratedMacroNames)) + { + // Failed to populate. + logger->log("Failed to populate macros for generated file: " + generatedFilePath.string(), ILogger::ELogSeverity::Error); + clang_disposeTranslationUnit(translationUnit); + return false; + } + } + + clang_disposeTranslationUnit(translationUnit); + + return true; +} + bool FileParser::parse(fs::path const& toParseFile, FileParsingResult& out_result, const kodgen::MacroCodeGenUnitSettings* codeGenSettings) noexcept { assert(_settings.use_count() != 0); @@ -71,51 +109,15 @@ bool FileParser::parse(fs::path const& toParseFile, FileParsingResult& out_resul //Fill the parsed file info out_result.parsedFile = FilesystemHelpers::sanitizePath(toParseFile); - // Prepare arrays for errors. - std::set notFoundGeneratedMacroNames; - std::vector errors; - - // Prepare variables for parsing. - bool rerunParsing = false; - bool populatedGeneratedFileWithMacros = false; - CXTranslationUnit translationUnit; - - do - { - rerunParsing = false; - - // Parse the given file. - translationUnit = clang_parseTranslationUnit(_clangIndex, toParseFile.string().c_str(), _settings->getCompilationArguments().data(), static_cast(_settings->getCompilationArguments().size()), nullptr, 0, CXTranslationUnit_SkipFunctionBodies | CXTranslationUnit_Incomplete | CXTranslationUnit_KeepGoing); - if (translationUnit != nullptr) - { - errors = getErrors(toParseFile, translationUnit, codeGenSettings, notFoundGeneratedMacroNames); - if (!populatedGeneratedFileWithMacros && !notFoundGeneratedMacroNames.empty()) - { - const auto generatedFilePath = codeGenSettings->getOutputDirectory() / codeGenSettings->getGeneratedHeaderFileName(toParseFile); - if (!populateFileWithMacros(generatedFilePath, notFoundGeneratedMacroNames)) - { - out_result.errors.emplace_back("Failed to populate the generated file " + generatedFilePath.string() + " with macros."); - postParse(toParseFile, out_result); - return false; - } - - notFoundGeneratedMacroNames.clear(); - populatedGeneratedFileWithMacros = true; - rerunParsing = true; - } - else if (populatedGeneratedFileWithMacros) - { - // Clear generated file as it will be filled with an actual information. - const auto generatedFilePath = codeGenSettings->getOutputDirectory() / codeGenSettings->getGeneratedHeaderFileName(toParseFile); - std::ofstream file(generatedFilePath); // truncate the file - file.close(); - } - } - }while(rerunParsing); + // Parse the given file. + auto translationUnit = clang_parseTranslationUnit(_clangIndex, toParseFile.string().c_str(), _settings->getCompilationArguments().data(), static_cast(_settings->getCompilationArguments().size()), nullptr, 0, CXTranslationUnit_SkipFunctionBodies | CXTranslationUnit_Incomplete | CXTranslationUnit_KeepGoing); if (translationUnit != nullptr) { - if (errors.empty()) + std::set notFoundGeneratedMacroNames; + const auto errors = getErrors(toParseFile, translationUnit, codeGenSettings, notFoundGeneratedMacroNames); + + if (errors.empty() && notFoundGeneratedMacroNames.empty()) { ParsingContext& context = pushContext(translationUnit, out_result); @@ -140,8 +142,6 @@ bool FileParser::parse(fs::path const& toParseFile, FileParsingResult& out_resul { logDiagnostic(translationUnit); } - - clang_disposeTranslationUnit(translationUnit); } else { @@ -149,7 +149,13 @@ bool FileParser::parse(fs::path const& toParseFile, FileParsingResult& out_resul { out_result.errors.emplace_back(message); } + for (const auto& macroName : notFoundGeneratedMacroNames) + { + out_result.errors.emplace_back("Unknown macro: " + macroName); + } } + + clang_disposeTranslationUnit(translationUnit); } else { @@ -334,10 +340,10 @@ void FileParser::postParse(fs::path const&, FileParsingResult const&) noexcept */ } -std::pair FileParser::splitClassFooterMacroPattern(const std::string& classFooterMacroPattern) +std::pair FileParser::splitMacroPattern(const std::string& macroPattern) { // Get left text. - const auto leftSharpPos = classFooterMacroPattern.find('#'); + const auto leftSharpPos = macroPattern.find('#'); if (leftSharpPos == std::string::npos) { return std::make_pair("", ""); @@ -345,19 +351,19 @@ std::pair FileParser::splitClassFooterMacroPattern(con std::string leftText; if (leftSharpPos != 0) { - leftText = classFooterMacroPattern.substr(0, leftSharpPos); + leftText = macroPattern.substr(0, leftSharpPos); } // Get right text. - const auto rightSharpPos = classFooterMacroPattern.rfind('#'); + const auto rightSharpPos = macroPattern.rfind('#'); if (rightSharpPos == std::string::npos) { return std::make_pair("", ""); } std::string rightText; - if (rightSharpPos != classFooterMacroPattern.size()) + if (rightSharpPos != macroPattern.size()) { - rightText = classFooterMacroPattern.substr(rightSharpPos + 1); + rightText = macroPattern.substr(rightSharpPos + 1); } return std::make_pair(leftText, rightText); @@ -373,7 +379,7 @@ std::vector FileParser::getErrors( const unsigned int diagnosticsCount = clang_getNumDiagnosticsInSet(diagnostics); const std::string generatedHeaderFilename = codeGenSettings->getGeneratedHeaderFileName(toParseFile).filename().string(); const std::string fileGeneratedMacroName = codeGenSettings->getHeaderFileFooterMacro(toParseFile); - const auto [leftClassFooterMacroText, rightClassFooterMacroText] = splitClassFooterMacroPattern(codeGenSettings->getClassFooterMacroPattern()); + const auto [leftClassFooterMacroText, rightClassFooterMacroText] = splitMacroPattern(codeGenSettings->getClassFooterMacroPattern()); if (leftClassFooterMacroText.empty() && rightClassFooterMacroText.empty()) { return {"failed to split class footer macro pattern"}; From 72149a13d3725da8d12a5f8f9992f88bd04b6b70 Mon Sep 17 00:00:00 2001 From: Aleksandr Tretyakov Date: Sat, 3 Sep 2022 22:08:39 +0500 Subject: [PATCH 08/12] Disable error limit so our pre-parsing step will not skip anything. --- Kodgen/Source/Parsing/ParsingSettings.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Kodgen/Source/Parsing/ParsingSettings.cpp b/Kodgen/Source/Parsing/ParsingSettings.cpp index 8ec70821..af0d80f6 100644 --- a/Kodgen/Source/Parsing/ParsingSettings.cpp +++ b/Kodgen/Source/Parsing/ParsingSettings.cpp @@ -104,6 +104,9 @@ void ParsingSettings::refreshCompilationArguments(ILogger* logger) noexcept //Parsing C++ _compilationArguments.emplace_back("-xc++"); + // Disable error limit so our pre-parsing step will not skip anything. + _compilationArguments.emplace_back("-ferror-limit=0"); + _compilationArguments.emplace_back(_additionalClangArguments.data()); #if KODGEN_DEV From 68c9b97350c8c1b792f7fe0ae352c631f125f03d Mon Sep 17 00:00:00 2001 From: Aleksandr Tretyakov Date: Sun, 4 Sep 2022 13:50:41 +0500 Subject: [PATCH 09/12] Rework parsing (+backwards compatibility). After I've decided to completely ignore the iterations thing (reruning all steps multiple times) I decided to add a backwards compatibility. I've added a new parameter in the `KodgenSettings.toml` - `shouldFailCodeGenerationOnClangErrors` which is `false` by default. When this setting is turned off we use the old parsing algorithm (existed before this branch, ignores parsing errors), when it's turned on we use the new approach where we fail on any Clang error that occur during the parsing. --- .../Include/Kodgen/CodeGen/CodeGenManager.h | 28 +++ .../Include/Kodgen/CodeGen/CodeGenManager.inl | 196 ++++++++++++------ Kodgen/Include/Kodgen/Parsing/FileParser.h | 21 +- .../Include/Kodgen/Parsing/ParsingSettings.h | 16 +- Kodgen/KodgenSettings.toml | 3 + Kodgen/Source/Parsing/FileParser.cpp | 61 +++++- Kodgen/Source/Parsing/ParsingSettings.cpp | 9 + 7 files changed, 269 insertions(+), 65 deletions(-) diff --git a/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.h b/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.h index e74208a2..5c01e387 100644 --- a/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.h +++ b/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.h @@ -42,6 +42,34 @@ namespace kodgen std::set const& toProcessFiles, CodeGenResult& out_genResult) noexcept; + /** + * @brief Process all provided files ignoring Clang parsing errors on multiple threads. + * + * @param fileParser Original file parser to use to parse registered files. A copy of this parser will be used for each generation thread. + * @param codeGenUnit Generation unit used to generate files. It must have a clean state when this method is called. + * @param toProcessFiles Collection of all files to process. + * @param out_genResult Reference to the generation result to fill during file generation. + */ + template + void processFilesIgnoreErrors(FileParserType& fileParser, + CodeGenUnitType& codeGenUnit, + std::set const& toProcessFiles, + CodeGenResult& out_genResult) noexcept; + + /** + * @brief Process all provided files and fail on any Clang parsing errors on multiple threads. + * + * @param fileParser Original file parser to use to parse registered files. A copy of this parser will be used for each generation thread. + * @param codeGenUnit Generation unit used to generate files. It must have a clean state when this method is called. + * @param toProcessFiles Collection of all files to process. + * @param out_genResult Reference to the generation result to fill during file generation. + */ + template + void processFilesFailOnErrors(FileParserType& fileParser, + CodeGenUnitType& codeGenUnit, + std::set const& toProcessFiles, + CodeGenResult& out_genResult) noexcept; + /** * @brief Identify all files which will be parsed & regenerated. * diff --git a/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl b/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl index 062c173a..eb037ece 100644 --- a/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl +++ b/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl @@ -7,88 +7,161 @@ template void CodeGenManager::processFiles(FileParserType& fileParser, CodeGenUnitType& codeGenUnit, std::set const& toProcessFiles, CodeGenResult& out_genResult) noexcept +{ + ParsingSettings parsingSettings = fileParser.getSettings(); + if (!parsingSettings.shouldFailCodeGenerationOnClangErrors) + { + processFilesIgnoreErrors(fileParser, codeGenUnit, toProcessFiles, out_genResult); + } + else + { + processFilesFailOnErrors(fileParser, codeGenUnit, toProcessFiles, out_genResult); + } +} + +template +void CodeGenManager::processFilesFailOnErrors(FileParserType& fileParser, CodeGenUnitType& codeGenUnit, std::set const& toProcessFiles, CodeGenResult& out_genResult) noexcept { std::vector> generationTasks; - std::vector> parsingTasks; - uint8 iterationCount = codeGenUnit.getIterationCount(); //Reserve enough space for all tasks - generationTasks.reserve(toProcessFiles.size() * iterationCount); + generationTasks.reserve(toProcessFiles.size()); //Launch all parsing -> generation processes std::shared_ptr parsingTask; std::shared_ptr preParsingTask; const kodgen::MacroCodeGenUnitSettings* codeGenSettings = codeGenUnit.getSettings(); + + //Lock the thread pool until all tasks have been pushed to avoid competing for the tasks mutex + _threadPool.setIsRunning(false); + + // First, run pre-parse step in which we will fill generated files + // with reflection macros (file/class macros). + // This will avoid the following issue: if we are using inheritance and we haven't + // generated parent's macros while parsing child class we will fail with an error. + for (fs::path const& file : toProcessFiles) + { + auto preParsingTaskLambda = [codeGenSettings, &fileParser, &file](TaskBase*) -> bool + { + FileParserType fileParserCopy = fileParser; + return fileParserCopy.prepareForParsing(file, codeGenSettings); + }; + + preParsingTask = _threadPool.submitTask(std::string("Pre-parsing ") + file.string(), preParsingTaskLambda); + } + + // Wait for pre-parse step to finish. + _threadPool.setIsRunning(true); + _threadPool.joinWorkers(); + _threadPool.setIsRunning(false); - for (int i = 0; i < iterationCount; i++) + // Run parsing step. + std::vector> parsingTasks; + for (fs::path const& file : toProcessFiles) { - //Lock the thread pool until all tasks have been pushed to avoid competing for the tasks mutex - _threadPool.setIsRunning(false); + auto parsingTaskLambda = [codeGenSettings, &fileParser, &file](TaskBase*) -> FileParsingResult + { + FileParsingResult parsingResult; + + // Copy a parser for this task. + FileParserType fileParserCopy = fileParser; - // First, run pre-parse step in which we will fill generated files - // with reflection macros (file/class macros). - // This will avoid the following issue: if we are using inheritance and we haven't - // generated parent's data while parsing child class we will fail with an error. - for (fs::path const& file : toProcessFiles) + fileParserCopy.parseFailOnErrors(file, parsingResult, codeGenSettings); + + return parsingResult; + }; + + //Add file to the list of parsed files before starting the task to avoid having to synchronize threads + out_genResult.parsedFiles.push_back(file); + + parsingTask = _threadPool.submitTask(std::string("Parsing ") + file.string(), parsingTaskLambda); + parsingTasks.push_back(parsingTask); + } + + // Wait for parse step to finish. + // We wait here for all tasks to be finished because on the next step + // (the generation step) we will fill generated files with an actual data + // and this will clear existing macro defines that generated files have, + // but we need these macros for all parsing tasks to be finished without errors. + _threadPool.setIsRunning(true); + _threadPool.joinWorkers(); + _threadPool.setIsRunning(false); + + // Run code generation after all files were parsed. + size_t parsingTaskIndex = 0; + for (fs::path const& file : toProcessFiles) + { + // Clear generated file as it will be filled with an actual information. + // Right now it has some defines that were used for proper parsing. + const auto generatedFilePath = codeGenSettings->getOutputDirectory() / codeGenSettings->getGeneratedHeaderFileName(file); + std::ofstream generatedFile(generatedFilePath); // truncate the file + generatedFile.close(); + + auto generationTaskLambda = [&codeGenUnit](TaskBase* parsingTask) -> CodeGenResult { - auto preParsingTaskLambda = [codeGenSettings, &fileParser, &file](TaskBase*) -> bool + CodeGenResult out_generationResult; + + //Copy the generation unit model to have a fresh one for this generation unit + CodeGenUnitType generationUnit = codeGenUnit; + + // Get the result of the parsing task. + FileParsingResult parsingResult = TaskHelper::getDependencyResult(parsingTask, 0u); + + //Generate the file if no errors occured during parsing + if (parsingResult.errors.empty()) { - FileParserType fileParserCopy = fileParser; - return fileParserCopy.prepareForParsing(file, codeGenSettings); - }; + out_generationResult.completed = generationUnit.generateCode(parsingResult); + } - preParsingTask = _threadPool.submitTask(std::string("Pre-parsing ") + std::to_string(i), preParsingTaskLambda); - } + return out_generationResult; + }; + + generationTasks.emplace_back(_threadPool.submitTask(std::string("Generation ") + file.string(), generationTaskLambda, { parsingTasks[parsingTaskIndex] })); + parsingTaskIndex += 1; + } - // Wait for pre-parse step to finish. - _threadPool.setIsRunning(true); - _threadPool.joinWorkers(); + // Wait for code generation. + _threadPool.setIsRunning(true); + _threadPool.joinWorkers(); + + //Merge all generation results together + for (std::shared_ptr& task : generationTasks) + { + out_genResult.mergeResult(TaskHelper::getResult(task.get())); + } +} + +template +void CodeGenManager::processFilesIgnoreErrors(FileParserType& fileParser, CodeGenUnitType& codeGenUnit, std::set const& toProcessFiles, CodeGenResult& out_genResult) noexcept +{ + std::vector> generationTasks; + uint8 iterationCount = codeGenUnit.getIterationCount(); + + //Reserve enough space for all tasks + generationTasks.reserve(toProcessFiles.size() * iterationCount); + + //Launch all parsing -> generation processes + std::shared_ptr parsingTask; + + for (int i = 0; i < iterationCount; i++) + { + //Lock the thread pool until all tasks have been pushed to avoid competing for the tasks mutex _threadPool.setIsRunning(false); - // Run parsing step. for (fs::path const& file : toProcessFiles) { - auto parsingTaskLambda = [codeGenSettings, &fileParser, &file](TaskBase*) -> FileParsingResult + auto parsingTaskLambda = [&fileParser, &file](TaskBase*) -> FileParsingResult { - FileParsingResult parsingResult; - //Copy a parser for this task FileParserType fileParserCopy = fileParser; + FileParsingResult parsingResult; - fileParserCopy.parse(file, parsingResult, codeGenSettings); + fileParserCopy.parseIgnoreErrors(file, parsingResult); return parsingResult; }; - //Add file to the list of parsed files before starting the task to avoid having to synchronize threads - out_genResult.parsedFiles.push_back(file); - - //Parse files - //For multiple iterations on a same file, the parsing task depends on the previous generation task for the same file - parsingTask = _threadPool.submitTask(std::string("Parsing ") + std::to_string(i), parsingTaskLambda); - parsingTasks.push_back(parsingTask); - } - - // Wait for parse step to finish. - // We wait here for all tasks to be finished because on the next step - // (the generation step) we will fill generated files with an actual data - // and this will clear existing macro defines that generated files have, - // but we need these macros for all parsing tasks to be finished without errors. - _threadPool.setIsRunning(true); - _threadPool.joinWorkers(); - _threadPool.setIsRunning(false); - - // Run code generation after all files were parsed. - size_t parsingTaskIndex = 0; - for (fs::path const& file : toProcessFiles) - { - // Clear generated file as it will be filled with an actual information. - // Right now it has some defines that were used for proper parsing. - const auto generatedFilePath = codeGenSettings->getOutputDirectory() / codeGenSettings->getGeneratedHeaderFileName(file); - std::ofstream generatedFile(generatedFilePath); // truncate the file - generatedFile.close(); - auto generationTaskLambda = [&codeGenUnit](TaskBase* parsingTask) -> CodeGenResult { CodeGenResult out_generationResult; @@ -107,15 +180,22 @@ void CodeGenManager::processFiles(FileParserType& fileParser, CodeGenUnitType& c return out_generationResult; }; - - generationTasks.emplace_back(_threadPool.submitTask(std::string("Generation ") + std::to_string(i), generationTaskLambda, { parsingTasks[parsingTaskIndex] })); - parsingTaskIndex += 1; + + //Add file to the list of parsed files before starting the task to avoid having to synchronize threads + out_genResult.parsedFiles.push_back(file); + + //Parse files + //For multiple iterations on a same file, the parsing task depends on the previous generation task for the same file + parsingTask = _threadPool.submitTask(std::string("Parsing ") + std::to_string(i), parsingTaskLambda); + + //Generate code + generationTasks.emplace_back(_threadPool.submitTask(std::string("Generation ") + std::to_string(i), generationTaskLambda, { parsingTask })); } - // Wait for code generation. + //Wait for this iteration to complete before continuing any further + //(an iteration N depends on the iteration N - 1) _threadPool.setIsRunning(true); _threadPool.joinWorkers(); - parsingTasks.clear(); } //Merge all generation results together diff --git a/Kodgen/Include/Kodgen/Parsing/FileParser.h b/Kodgen/Include/Kodgen/Parsing/FileParser.h index b14fb6f8..22b6ee53 100644 --- a/Kodgen/Include/Kodgen/Parsing/FileParser.h +++ b/Kodgen/Include/Kodgen/Parsing/FileParser.h @@ -185,7 +185,7 @@ namespace kodgen virtual ~FileParser() noexcept; /** - * @brief Prepares initial generated file for parsing. + * @brief Prepares initial generated file for parsing before calling to @ref parseFailOnErrors. * * @param toParseFile Path to the file to parse. * @param codeGenSettings Code generation settings. @@ -196,7 +196,18 @@ namespace kodgen const kodgen::MacroCodeGenUnitSettings* codeGenSettings) noexcept; /** - * @brief Parse the file and fill the FileParsingResult. + * @brief Parse the file and fill the FileParsingResult while ignoring any parsing errors. + * + * @param toParseFile Path to the file to parse. + * @param out_result Result filled while parsing the file. + * + * @return true if the parsing process finished without error, else false + */ + bool parseIgnoreErrors(fs::path const& toParseFile, + FileParsingResult& out_result) noexcept; + + /** + * @brief Parse the file and fill the FileParsingResult while failing on any parsing errors. * * @param toParseFile Path to the file to parse. * @param out_result Result filled while parsing the file. @@ -204,9 +215,9 @@ namespace kodgen * * @return true if the parsing process finished without error, else false */ - bool parse(fs::path const& toParseFile, - FileParsingResult& out_result, - const kodgen::MacroCodeGenUnitSettings* codeGenSettings) noexcept; + bool parseFailOnErrors(fs::path const& toParseFile, + FileParsingResult& out_result, + const kodgen::MacroCodeGenUnitSettings* codeGenSettings) noexcept; /** * @brief Getter for _settings field. diff --git a/Kodgen/Include/Kodgen/Parsing/ParsingSettings.h b/Kodgen/Include/Kodgen/Parsing/ParsingSettings.h index 85212ecb..a9c12b00 100644 --- a/Kodgen/Include/Kodgen/Parsing/ParsingSettings.h +++ b/Kodgen/Include/Kodgen/Parsing/ParsingSettings.h @@ -132,7 +132,16 @@ namespace kodgen * @param logger Optional logger used to issue loading logs. Can be nullptr. */ void loadAdditionalClangArguments(toml::value const& parsingSettings, - ILogger* logger) noexcept; + ILogger* logger) noexcept; + + /** + * @brief Load the shouldFailCodeGenerationOnClangErrors setting from toml. + * + * @param parsingSettings Toml content. + * @param logger Optional logger used to issue loading logs. Can be nullptr. + */ + void loadShouldFailCodeGenerationOnClangErrors(toml::value const& parsingSettings, + ILogger* logger) noexcept; /** * @brief Load the _projectIncludeDirectories setting from toml. @@ -198,6 +207,11 @@ namespace kodgen */ bool shouldLogDiagnostic = false; + /** + * Whether to fail code generation on any parsing errors or not. + */ + bool shouldFailCodeGenerationOnClangErrors = false; + virtual ~ParsingSettings() = default; /** diff --git a/Kodgen/KodgenSettings.toml b/Kodgen/KodgenSettings.toml index 97bfed10..c521de4f 100644 --- a/Kodgen/KodgenSettings.toml +++ b/Kodgen/KodgenSettings.toml @@ -57,6 +57,9 @@ shouldParseAllMethods = false shouldParseAllEnums = false shouldParseAllEnumValues = true +# Ignores parsing errors, may lead to incorrect type information for non-reflected types. +shouldFailCodeGenerationOnClangErrors = false + shouldLogDiagnostic = false propertySeparator = "," diff --git a/Kodgen/Source/Parsing/FileParser.cpp b/Kodgen/Source/Parsing/FileParser.cpp index ae8cc979..2fb91e34 100644 --- a/Kodgen/Source/Parsing/FileParser.cpp +++ b/Kodgen/Source/Parsing/FileParser.cpp @@ -96,7 +96,7 @@ bool FileParser::prepareForParsing(fs::path const& toParseFile, const kodgen::Ma return true; } -bool FileParser::parse(fs::path const& toParseFile, FileParsingResult& out_result, const kodgen::MacroCodeGenUnitSettings* codeGenSettings) noexcept +bool FileParser::parseFailOnErrors(fs::path const& toParseFile, FileParsingResult& out_result, const kodgen::MacroCodeGenUnitSettings* codeGenSettings) noexcept { assert(_settings.use_count() != 0); @@ -172,6 +172,65 @@ bool FileParser::parse(fs::path const& toParseFile, FileParsingResult& out_resul return isSuccess; } +bool FileParser::parseIgnoreErrors(fs::path const& toParseFile, FileParsingResult& out_result) noexcept +{ + assert(_settings.use_count() != 0); + + bool isSuccess = false; + + preParse(toParseFile); + + if (fs::exists(toParseFile) && !fs::is_directory(toParseFile)) + { + //Fill the parsed file info + out_result.parsedFile = FilesystemHelpers::sanitizePath(toParseFile); + + //Parse the given file + CXTranslationUnit translationUnit = clang_parseTranslationUnit(_clangIndex, toParseFile.string().c_str(), _settings->getCompilationArguments().data(), static_cast(_settings->getCompilationArguments().size()), nullptr, 0, CXTranslationUnit_SkipFunctionBodies | CXTranslationUnit_Incomplete | CXTranslationUnit_KeepGoing); + + if (translationUnit != nullptr) + { + ParsingContext& context = pushContext(translationUnit, out_result); + + if (clang_visitChildren(context.rootCursor, &FileParser::parseNestedEntity, this) || !out_result.errors.empty()) + { + //ERROR + } + else + { + //Refresh all outer entities contained in the final result + refreshOuterEntity(out_result); + + isSuccess = true; + } + + popContext(); + + //There should not have any context left once parsing has finished + assert(contextsStack.empty()); + + if (_settings->shouldLogDiagnostic) + { + logDiagnostic(translationUnit); + } + + clang_disposeTranslationUnit(translationUnit); + } + else + { + out_result.errors.emplace_back("Failed to initialize translation unit for file: " + toParseFile.string()); + } + } + else + { + out_result.errors.emplace_back("File " + toParseFile.string() + " doesn't exist."); + } + + postParse(toParseFile, out_result); + + return isSuccess; +} + CXChildVisitResult FileParser::parseNestedEntity(CXCursor cursor, CXCursor /* parentCursor */, CXClientData clientData) noexcept { FileParser* parser = reinterpret_cast(clientData); diff --git a/Kodgen/Source/Parsing/ParsingSettings.cpp b/Kodgen/Source/Parsing/ParsingSettings.cpp index af0d80f6..49a4c412 100644 --- a/Kodgen/Source/Parsing/ParsingSettings.cpp +++ b/Kodgen/Source/Parsing/ParsingSettings.cpp @@ -145,6 +145,7 @@ bool ParsingSettings::loadSettingsValues(toml::value const& tomlData, ILogger* l loadShouldParseAllEntities(tomlParsingSettings, logger); loadShouldAbortParsingOnFirstError(tomlParsingSettings, logger); loadShouldLogDiagnostic(tomlParsingSettings, logger); + loadShouldFailCodeGenerationOnClangErrors(tomlParsingSettings, logger); loadCompilerExeName(tomlParsingSettings, logger); loadProjectIncludeDirectories(tomlParsingSettings, logger); loadAdditionalClangArguments(tomlParsingSettings, logger); @@ -248,6 +249,14 @@ void ParsingSettings::loadAdditionalClangArguments(toml::value const& tomlFilePa } } +void ParsingSettings::loadShouldFailCodeGenerationOnClangErrors(toml::value const& tomlFileParsingSettings, ILogger* logger) noexcept +{ + if (TomlUtility::updateSetting(tomlFileParsingSettings, "shouldFailCodeGenerationOnClangErrors", shouldFailCodeGenerationOnClangErrors, logger) && logger != nullptr) + { + logger->log("[TOML] Load shouldFailCodeGenerationOnClangErrors: " + Helpers::toString(shouldFailCodeGenerationOnClangErrors)); + } +} + void ParsingSettings::loadShouldLogDiagnostic(toml::value const& tomlFileParsingSettings, ILogger* logger) noexcept { if (TomlUtility::updateSetting(tomlFileParsingSettings, "shouldLogDiagnostic", shouldLogDiagnostic, logger) && logger != nullptr) From 0affeddb9eb06836ff65cd3c770bddd6affa9c37 Mon Sep 17 00:00:00 2001 From: Aleksandr Tretyakov Date: Sun, 4 Sep 2022 17:41:10 +0500 Subject: [PATCH 10/12] Retry parsing failed files after all files. While running parsing for the tests I've noticed that there is a situation when 1 GENERATED macro is not captured in the pre-parsing step which fails the parsing step. I also noticed that this error might not occur due to the fact that pre-parsing step writes macros in parallel. Thus, I've made it so that the pre-parsing step writes found GENERATED macros in a single-threaded manner after the pre-parsing step but before the parsing step. Although this is indeed an improvement, it does not solved the issue where 1 GENERATED macro was not found. I then remembered about "iterations" thing and added a cycle. So now, files that failed the parsing step (only failed files) will be queued to be parsed again on the next cycle iteration. With this change, when we rerun the pre-parsing step for failed files we have enough info to find all GENERATED macros. Of cource I added a check where if the number of failed files stays the same we break out of the cycle and show an error. --- .../Include/Kodgen/CodeGen/CodeGenManager.inl | 225 ++++++++++++------ Kodgen/Include/Kodgen/Parsing/FileParser.h | 18 +- Kodgen/Source/Parsing/FileParser.cpp | 83 +++---- 3 files changed, 187 insertions(+), 139 deletions(-) diff --git a/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl b/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl index eb037ece..945ce558 100644 --- a/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl +++ b/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl @@ -32,99 +32,178 @@ void CodeGenManager::processFilesFailOnErrors(FileParserType& fileParser, CodeGe std::shared_ptr preParsingTask; const kodgen::MacroCodeGenUnitSettings* codeGenSettings = codeGenUnit.getSettings(); + std::set filesLeftToProcess = toProcessFiles; + std::vector parsingResultsOfFailedFiles; + size_t filesLeftBefore = 0; + + // Process files in cycle. + // Files that failed parsing step will be queued for the next cycle iteration to be parsed again. + // This is needed because sometimes not all GENERATED macros are filled on pre-parsing step (usually, + // when we have an include chain of multiple files that use reflection some GENERATED macros + // are not detected on pre-parsing step). + do + { + parsingResultsOfFailedFiles.clear(); + filesLeftBefore = filesLeftToProcess.size(); + std::set filesToProcessThisIteration = std::move(filesLeftToProcess); + + //Lock the thread pool until all tasks have been pushed to avoid competing for the tasks mutex + _threadPool.setIsRunning(false); - //Lock the thread pool until all tasks have been pushed to avoid competing for the tasks mutex - _threadPool.setIsRunning(false); + // First, run pre-parse step in which we will fill generated files + // with reflection macros (file/class macros). + // This will avoid the following issue: if we are using inheritance and we haven't + // generated parent's macros while parsing child class we will fail with an error. + std::vector> fileMacrosToDefine(filesToProcessThisIteration.size()); + size_t iPreParsingFileIndex = 0; + for (fs::path const& file : filesToProcessThisIteration) + { + std::set* macrosToDefine = &fileMacrosToDefine[iPreParsingFileIndex]; + auto preParsingTaskLambda = [codeGenSettings, &fileParser, &file, macrosToDefine](TaskBase*) -> bool + { + FileParserType fileParserCopy = fileParser; + return fileParserCopy.prepareForParsing(file, codeGenSettings, *macrosToDefine); + }; - // First, run pre-parse step in which we will fill generated files - // with reflection macros (file/class macros). - // This will avoid the following issue: if we are using inheritance and we haven't - // generated parent's macros while parsing child class we will fail with an error. - for (fs::path const& file : toProcessFiles) - { - auto preParsingTaskLambda = [codeGenSettings, &fileParser, &file](TaskBase*) -> bool + preParsingTask = _threadPool.submitTask(std::string("Pre-parsing ") + file.string(), preParsingTaskLambda); + + iPreParsingFileIndex += 1; + } + + // Wait for pre-parse step to finish. + _threadPool.setIsRunning(true); + _threadPool.joinWorkers(); + _threadPool.setIsRunning(false); + + // Define generated macros. + iPreParsingFileIndex = 0; + for (fs::path const& file : filesToProcessThisIteration) { - FileParserType fileParserCopy = fileParser; - return fileParserCopy.prepareForParsing(file, codeGenSettings); - }; + if (!fileMacrosToDefine[iPreParsingFileIndex].empty()) + { + // Populate generated file with macros. + const auto generatedFilePath = codeGenSettings->getOutputDirectory() / codeGenSettings->getGeneratedHeaderFileName(file); - preParsingTask = _threadPool.submitTask(std::string("Pre-parsing ") + file.string(), preParsingTaskLambda); - } + std::ofstream generatedfile(generatedFilePath, std::ios::app); + for (const auto& macroName : fileMacrosToDefine[iPreParsingFileIndex]) + { + generatedfile << "#define " + macroName + " " << std::endl; + } + generatedfile.close(); + } - // Wait for pre-parse step to finish. - _threadPool.setIsRunning(true); - _threadPool.joinWorkers(); - _threadPool.setIsRunning(false); - - // Run parsing step. - std::vector> parsingTasks; - for (fs::path const& file : toProcessFiles) - { - auto parsingTaskLambda = [codeGenSettings, &fileParser, &file](TaskBase*) -> FileParsingResult + iPreParsingFileIndex += 1; + } + + // Run parsing step. + std::vector> parsingTasks; + for (fs::path const& file : filesToProcessThisIteration) { - FileParsingResult parsingResult; + auto parsingTaskLambda = [codeGenSettings, &fileParser, &file, &filesLeftToProcess, &parsingResultsOfFailedFiles](TaskBase*) -> FileParsingResult + { + FileParsingResult parsingResult; - // Copy a parser for this task. - FileParserType fileParserCopy = fileParser; + // Copy a parser for this task. + FileParserType fileParserCopy = fileParser; - fileParserCopy.parseFailOnErrors(file, parsingResult, codeGenSettings); + fileParserCopy.parseFailOnErrors(file, parsingResult, codeGenSettings); + if (!parsingResult.errors.empty()) + { + for (const auto& error : parsingResult.errors) + { + parsingResultsOfFailedFiles.push_back(error); + } + parsingResult.errors.clear(); + filesLeftToProcess.insert(file); + } - return parsingResult; - }; + return parsingResult; + }; - //Add file to the list of parsed files before starting the task to avoid having to synchronize threads - out_genResult.parsedFiles.push_back(file); + //Add file to the list of parsed files before starting the task to avoid having to synchronize threads + out_genResult.parsedFiles.push_back(file); - parsingTask = _threadPool.submitTask(std::string("Parsing ") + file.string(), parsingTaskLambda); - parsingTasks.push_back(parsingTask); - } + parsingTask = _threadPool.submitTask(std::string("Parsing ") + file.string(), parsingTaskLambda); + parsingTasks.push_back(parsingTask); + } - // Wait for parse step to finish. - // We wait here for all tasks to be finished because on the next step - // (the generation step) we will fill generated files with an actual data - // and this will clear existing macro defines that generated files have, - // but we need these macros for all parsing tasks to be finished without errors. - _threadPool.setIsRunning(true); - _threadPool.joinWorkers(); - _threadPool.setIsRunning(false); - - // Run code generation after all files were parsed. - size_t parsingTaskIndex = 0; - for (fs::path const& file : toProcessFiles) - { - // Clear generated file as it will be filled with an actual information. - // Right now it has some defines that were used for proper parsing. - const auto generatedFilePath = codeGenSettings->getOutputDirectory() / codeGenSettings->getGeneratedHeaderFileName(file); - std::ofstream generatedFile(generatedFilePath); // truncate the file - generatedFile.close(); - - auto generationTaskLambda = [&codeGenUnit](TaskBase* parsingTask) -> CodeGenResult + // Wait for parse step to finish. + // We wait here for all tasks to be finished because on the next step + // (the generation step) we will fill generated files with an actual data + // and this will clear existing macro defines that generated files have, + // but we need these macros for all parsing tasks to be finished without errors. + _threadPool.setIsRunning(true); + _threadPool.joinWorkers(); + _threadPool.setIsRunning(false); + + // Run code generation after all files were parsed. + size_t parsingTaskIndex = 0; + for (fs::path const& file : filesToProcessThisIteration) { - CodeGenResult out_generationResult; + // Check if the parsing step failed for this file. + bool bParsingFailed = false; + for (const auto& failedFile : filesLeftToProcess) + { + if (failedFile == file) + { + bParsingFailed = true; + break; + } + } + if (bParsingFailed) + { + parsingTaskIndex += 1; + continue; + } + + // Clear generated file as it will be filled with an actual information. + // Right now it has some defines that were used for proper parsing. + const auto generatedFilePath = codeGenSettings->getOutputDirectory() / codeGenSettings->getGeneratedHeaderFileName(file); + std::ofstream generatedFile(generatedFilePath); // truncate the file + generatedFile.close(); + + auto generationTaskLambda = [&codeGenUnit](TaskBase* parsingTask) -> CodeGenResult + { + CodeGenResult out_generationResult; - //Copy the generation unit model to have a fresh one for this generation unit - CodeGenUnitType generationUnit = codeGenUnit; + // Copy the generation unit model to have a fresh one for this generation unit. + CodeGenUnitType generationUnit = codeGenUnit; - // Get the result of the parsing task. - FileParsingResult parsingResult = TaskHelper::getDependencyResult(parsingTask, 0u); + // Get the result of the parsing task. + FileParsingResult parsingResult = TaskHelper::getDependencyResult(parsingTask, 0u); - //Generate the file if no errors occured during parsing - if (parsingResult.errors.empty()) - { - out_generationResult.completed = generationUnit.generateCode(parsingResult); - } + // Generate the file if no errors occured during parsing. + if (parsingResult.errors.empty()) + { + out_generationResult.completed = generationUnit.generateCode(parsingResult); + } - return out_generationResult; - }; + return out_generationResult; + }; - generationTasks.emplace_back(_threadPool.submitTask(std::string("Generation ") + file.string(), generationTaskLambda, { parsingTasks[parsingTaskIndex] })); - parsingTaskIndex += 1; - } + generationTasks.emplace_back(_threadPool.submitTask(std::string("Generation ") + file.string(), generationTaskLambda, { parsingTasks[parsingTaskIndex] })); + parsingTaskIndex += 1; + } - // Wait for code generation. - _threadPool.setIsRunning(true); - _threadPool.joinWorkers(); + // Wait for code generation. + _threadPool.setIsRunning(true); + _threadPool.joinWorkers(); + }while(!filesLeftToProcess.empty() && filesLeftBefore != filesLeftToProcess.size()); + // Log errors. + if (logger != nullptr) + { + if (!parsingResultsOfFailedFiles.empty()) + { + out_genResult.completed = false; + } + + for (kodgen::ParsingError const& error : parsingResultsOfFailedFiles) + { + logger->log(error.toString(), kodgen::ILogger::ELogSeverity::Error); + } + } + //Merge all generation results together for (std::shared_ptr& task : generationTasks) { diff --git a/Kodgen/Include/Kodgen/Parsing/FileParser.h b/Kodgen/Include/Kodgen/Parsing/FileParser.h index 22b6ee53..2aa9c23f 100644 --- a/Kodgen/Include/Kodgen/Parsing/FileParser.h +++ b/Kodgen/Include/Kodgen/Parsing/FileParser.h @@ -120,16 +120,6 @@ namespace kodgen * @return a pair of strings between "##...##" substring. */ static std::pair splitMacroPattern(const std::string& macroPattern); - - /** - * Clears the file and adds "#define" statements for the specified macros. - * - * @param filePath File in which to define macros. - * @param macroNamesToDefine Macro names to define. - * - * @retun true if successfull, false is failed. - */ - static bool populateFileWithMacros(const fs::path& filePath, const std::set& macroNamesToDefine); /** * @brief Returns all errors found during translation unit parsing. @@ -187,13 +177,15 @@ namespace kodgen /** * @brief Prepares initial generated file for parsing before calling to @ref parseFailOnErrors. * - * @param toParseFile Path to the file to parse. - * @param codeGenSettings Code generation settings. + * @param toParseFile Path to the file to parse. + * @param codeGenSettings Code generation settings. + * @param notFoundGeneratedMacroNames Array of generated macros that needs to be defines. * * @return true if the parsing process finished without error, else false */ bool prepareForParsing(fs::path const& toParseFile, - const kodgen::MacroCodeGenUnitSettings* codeGenSettings) noexcept; + const kodgen::MacroCodeGenUnitSettings* codeGenSettings, + std::set& notFoundGeneratedMacroNames) noexcept; /** * @brief Parse the file and fill the FileParsingResult while ignoring any parsing errors. diff --git a/Kodgen/Source/Parsing/FileParser.cpp b/Kodgen/Source/Parsing/FileParser.cpp index 2fb91e34..c4d72d79 100644 --- a/Kodgen/Source/Parsing/FileParser.cpp +++ b/Kodgen/Source/Parsing/FileParser.cpp @@ -40,56 +40,23 @@ FileParser::~FileParser() noexcept } } -bool FileParser::populateFileWithMacros(const fs::path& filePath, const std::set& macroNamesToDefine) -{ - std::ofstream generatedfile(filePath, std::ios::app); - if (!generatedfile.is_open()) - { - return false; - } - - for (const auto& macroName : macroNamesToDefine) - { - generatedfile << "#define " + macroName + " " << std::endl; - } - - generatedfile.close(); - - return true; -} - -bool FileParser::prepareForParsing(fs::path const& toParseFile, const kodgen::MacroCodeGenUnitSettings* codeGenSettings) noexcept +bool FileParser::prepareForParsing(fs::path const& toParseFile, const kodgen::MacroCodeGenUnitSettings* codeGenSettings, std::set& notFoundGeneratedMacroNames) noexcept { assert(_settings.use_count() != 0); - logger->log("Starting pre-parsing step for " + toParseFile.string(), ILogger::ELogSeverity::Info); - if (!fs::exists(toParseFile) || fs::is_directory(toParseFile)) return false; // Do initial parsing. CXTranslationUnit translationUnit = clang_parseTranslationUnit(_clangIndex, toParseFile.string().c_str(), _settings->getCompilationArguments().data(), static_cast(_settings->getCompilationArguments().size()), nullptr, 0, CXTranslationUnit_SkipFunctionBodies | CXTranslationUnit_Incomplete | CXTranslationUnit_KeepGoing); if (!translationUnit) - { + { logger->log("Failed to initialize translation unit for file: " + toParseFile.string(), ILogger::ELogSeverity::Error); return false; } // Process errors. - std::set notFoundGeneratedMacroNames; + notFoundGeneratedMacroNames.clear(); const auto errors = getErrors(toParseFile, translationUnit, codeGenSettings, notFoundGeneratedMacroNames); - - if (!notFoundGeneratedMacroNames.empty()) - { - // Populate generated file with macros. - const auto generatedFilePath = codeGenSettings->getOutputDirectory() / codeGenSettings->getGeneratedHeaderFileName(toParseFile); - if (!populateFileWithMacros(generatedFilePath, notFoundGeneratedMacroNames)) - { - // Failed to populate. - logger->log("Failed to populate macros for generated file: " + generatedFilePath.string(), ILogger::ELogSeverity::Error); - clang_disposeTranslationUnit(translationUnit); - return false; - } - } clang_disposeTranslationUnit(translationUnit); @@ -102,8 +69,6 @@ bool FileParser::parseFailOnErrors(fs::path const& toParseFile, FileParsingResul bool isSuccess = false; - preParse(toParseFile); - if (fs::exists(toParseFile) && !fs::is_directory(toParseFile)) { //Fill the parsed file info @@ -167,7 +132,13 @@ bool FileParser::parseFailOnErrors(fs::path const& toParseFile, FileParsingResul out_result.errors.emplace_back("File " + toParseFile.string() + " doesn't exist."); } - postParse(toParseFile, out_result); + if (out_result.errors.empty()) + { + logger->log(toParseFile.string() + ": Found " + std::to_string(out_result.namespaces.size()) + " namespace(s), " + + std::to_string(out_result.structs.size()) + " struct(s), " + + std::to_string(out_result.classes.size()) + " classe(s) and " + + std::to_string(out_result.enums.size()) + " enum(s).", kodgen::ILogger::ELogSeverity::Info); + } return isSuccess; } @@ -436,6 +407,7 @@ std::vector FileParser::getErrors( { const CXDiagnosticSet diagnostics = clang_getDiagnosticSetFromTU(translationUnit); const unsigned int diagnosticsCount = clang_getNumDiagnosticsInSet(diagnostics); + const std::string generatedHeaderFilename = codeGenSettings->getGeneratedHeaderFileName(toParseFile).filename().string(); const std::string fileGeneratedMacroName = codeGenSettings->getHeaderFileFooterMacro(toParseFile); const auto [leftClassFooterMacroText, rightClassFooterMacroText] = splitMacroPattern(codeGenSettings->getClassFooterMacroPattern()); @@ -443,18 +415,33 @@ std::vector FileParser::getErrors( { return {"failed to split class footer macro pattern"}; } - const std::string unknownTypeNameError = "unknown type name '"; + const auto [leftGeneratedHeaderPatternText, rightGeneratedHeaderPatternText] = splitMacroPattern(codeGenSettings->getGeneratedHeaderFileNamePattern()); + if (leftGeneratedHeaderPatternText.empty() && rightGeneratedHeaderPatternText.empty()) + { + return {"failed to split generated header file name pattern"}; + } + const std::string unknownTypeNameError = "unknown type name '"; std::vector errors; for (unsigned i = 0u; i < diagnosticsCount; i++) { const CXDiagnostic diagnostic(clang_getDiagnosticInSet(diagnostics, i)); - //auto diagnosticMessage = Helpers::getString(clang_formatDiagnostic(diagnostic, clang_defaultDiagnosticDisplayOptions())); auto diagnosticMessage = Helpers::getString(clang_getDiagnosticSpelling(diagnostic)); + // Prepare location information. + CXFile file; + unsigned line, column; + clang_getExpansionLocation( + clang_getDiagnosticLocation(diagnostic), + &file, + &line, + &column, + nullptr); + const auto errorFilePath = std::filesystem::path(Helpers::getString(clang_getFileName(file))).make_preferred(); + // Look if this error is related to GENERATED macros like "unknown type name 'File_MyClass_GENERATED'". - if (diagnosticMessage.find(unknownTypeNameError) != std::string::npos) + if (diagnosticMessage.find(unknownTypeNameError) != std::string::npos && errorFilePath == toParseFile) { const auto unknownTypeName = diagnosticMessage.substr( // don't include last ' character unknownTypeNameError.size(), diagnosticMessage.size() - 1 - unknownTypeNameError.size()); @@ -471,17 +458,7 @@ std::vector FileParser::getErrors( } } - // Prepare location information. - CXFile file; - unsigned line, column; - clang_getExpansionLocation( - clang_getDiagnosticLocation(diagnostic), - &file, - &line, - &column, - nullptr); - auto location = file ? Helpers::getString(clang_getFileName(file)) : std::string(); - location += ", line " + std::to_string(line) + ", column " + std::to_string(column) + ""; + const auto location = errorFilePath.string() + ", line " + std::to_string(line) + ", column " + std::to_string(column) + ""; errors.push_back(diagnosticMessage + " (" + location + ")"); clang_disposeDiagnostic(diagnostic); From 798781842c88429e5870c581cb0de21d5bf50c51 Mon Sep 17 00:00:00 2001 From: Aleksandr Tretyakov Date: Sun, 4 Sep 2022 18:42:28 +0500 Subject: [PATCH 11/12] Log file path that failed parsing. When a parsing error occurres we now not only show path to the file where error occurred but also path to the file that was parsed. This actually helps when we are parsing file A and it includes file B and a parsing error occurres in the file B. --- Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl b/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl index 945ce558..b6c4a007 100644 --- a/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl +++ b/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl @@ -33,7 +33,7 @@ void CodeGenManager::processFilesFailOnErrors(FileParserType& fileParser, CodeGe const kodgen::MacroCodeGenUnitSettings* codeGenSettings = codeGenUnit.getSettings(); std::set filesLeftToProcess = toProcessFiles; - std::vector parsingResultsOfFailedFiles; + std::vector> parsingResultsOfFailedFiles; size_t filesLeftBefore = 0; // Process files in cycle. @@ -111,7 +111,7 @@ void CodeGenManager::processFilesFailOnErrors(FileParserType& fileParser, CodeGe { for (const auto& error : parsingResult.errors) { - parsingResultsOfFailedFiles.push_back(error); + parsingResultsOfFailedFiles.push_back(std::make_pair(file, error)); } parsingResult.errors.clear(); filesLeftToProcess.insert(file); @@ -198,9 +198,9 @@ void CodeGenManager::processFilesFailOnErrors(FileParserType& fileParser, CodeGe out_genResult.completed = false; } - for (kodgen::ParsingError const& error : parsingResultsOfFailedFiles) + for (const auto error : parsingResultsOfFailedFiles) { - logger->log(error.toString(), kodgen::ILogger::ELogSeverity::Error); + logger->log("While processing the following file: " + error.first.string() + ": " + error.second.toString(), kodgen::ILogger::ELogSeverity::Error); } } From 701e6f6a08452f568411d969bc35228b2c9c6e70 Mon Sep 17 00:00:00 2001 From: Alexander Tretyakov Date: Sun, 4 Sep 2022 20:18:00 +0500 Subject: [PATCH 12/12] Don't copy value in `for` loop, use reference. --- Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl b/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl index b6c4a007..74c4c3b4 100644 --- a/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl +++ b/Kodgen/Include/Kodgen/CodeGen/CodeGenManager.inl @@ -198,7 +198,7 @@ void CodeGenManager::processFilesFailOnErrors(FileParserType& fileParser, CodeGe out_genResult.completed = false; } - for (const auto error : parsingResultsOfFailedFiles) + for (const auto& error : parsingResultsOfFailedFiles) { logger->log("While processing the following file: " + error.first.string() + ": " + error.second.toString(), kodgen::ILogger::ELogSeverity::Error); } @@ -325,4 +325,4 @@ CodeGenResult CodeGenManager::run(FileParserType& fileParser, CodeGenUnitType& c } return genResult; -} \ No newline at end of file +}