diff --git a/doc/10-icinga-template-library.md b/doc/10-icinga-template-library.md index b8775c74bce..b8ccce08722 100644 --- a/doc/10-icinga-template-library.md +++ b/doc/10-icinga-template-library.md @@ -83,6 +83,7 @@ Custom variables passed as [command parameters](03-monitoring-basics.md#command- Name | Description -----------------------|--------------- icinga\_min\_version | **Optional.** Required minimum Icinga 2 version, e.g. `2.8.0`. If not satisfied, the state changes to `Critical`. Release packages only. +icinga\_verbose | **Optional.** If the last reload failed, tell the reason. ### cluster diff --git a/lib/base/application.cpp b/lib/base/application.cpp index 89a0f55a2e3..6ec6b0ce85d 100644 --- a/lib/base/application.cpp +++ b/lib/base/application.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -30,6 +31,7 @@ #ifdef _WIN32 #include #else /* _WIN32 */ +#include "base/shared-memory.hpp" #include #endif /* _WIN32 */ @@ -63,11 +65,18 @@ char **Application::m_ArgV; double Application::m_StartTime; bool Application::m_ScriptDebuggerEnabled = false; +Application::LastReloadFailed* Application::m_LastReloadFailed (Application::AllocLastReloadFailed()); + +Application::LastReloadFailed* Application::AllocLastReloadFailed() +{ #ifdef _WIN32 -double Application::m_LastReloadFailed = 0; + static LastReloadFailed lrf; + return &lrf; #else /* _WIN32 */ -SharedMemory Application::m_LastReloadFailed (0); + static SharedMemory slrf; + return &slrf.Get(); #endif /* _WIN32 */ +} #ifdef _WIN32 static LPTOP_LEVEL_EXCEPTION_FILTER l_DefaultUnhandledExceptionFilter = nullptr; @@ -379,7 +388,7 @@ void Application::OnShutdown() static void ReloadProcessCallbackInternal(const ProcessResult& pr) { if (pr.ExitStatus != 0) { - Application::SetLastReloadFailed(Utility::GetTime()); + Application::SetLastReloadFailed(Utility::GetTime(), pr.Output); Log(LogCritical, "Application", "Found error in config: reloading aborted"); } #ifdef _WIN32 @@ -1211,22 +1220,17 @@ void Application::SetScriptDebuggerEnabled(bool enabled) m_ScriptDebuggerEnabled = enabled; } -double Application::GetLastReloadFailed() +std::pair Application::GetLastReloadFailed() { -#ifdef _WIN32 - return m_LastReloadFailed; -#else /* _WIN32 */ - return m_LastReloadFailed.Get().load(); -#endif /* _WIN32 */ + LastReloadFailed::SharedLock lock (m_LastReloadFailed->Mutex); + return {m_LastReloadFailed->When, String(m_LastReloadFailed->Why)}; } -void Application::SetLastReloadFailed(double ts) +void Application::SetLastReloadFailed(double ts, const String& error) { -#ifdef _WIN32 - m_LastReloadFailed = ts; -#else /* _WIN32 */ - m_LastReloadFailed.Get().store(ts); -#endif /* _WIN32 */ + LastReloadFailed::UniqueLock lock (m_LastReloadFailed->Mutex); + m_LastReloadFailed->When = ts; + strncpy(m_LastReloadFailed->Why, error.CStr(), sizeof(m_LastReloadFailed->Why)); } void Application::ValidateName(const Lazy& lvalue, const ValidationUtils& utils) diff --git a/lib/base/application.hpp b/lib/base/application.hpp index 0578b8aa8bc..8eef49c8000 100644 --- a/lib/base/application.hpp +++ b/lib/base/application.hpp @@ -4,12 +4,20 @@ #define APPLICATION_H #include "base/i2-base.hpp" -#include "base/atomic.hpp" #include "base/application-ti.hpp" #include "base/logger.hpp" #include "base/configuration.hpp" -#include "base/shared-memory.hpp" #include +#include + +#ifdef _WIN32 +#include +#include +#else /* _WIN32 */ +#include +#include +#include +#endif /* _WIN32 */ namespace icinga { @@ -102,8 +110,8 @@ class Application : public ObjectImpl { static bool GetScriptDebuggerEnabled(); static void SetScriptDebuggerEnabled(bool enabled); - static double GetLastReloadFailed(); - static void SetLastReloadFailed(double ts); + static std::pair GetLastReloadFailed(); + static void SetLastReloadFailed(double ts, const String& error); static void DisplayInfoMessage(std::ostream& os, bool skipVersion = false); @@ -139,13 +147,26 @@ class Application : public ObjectImpl { static double m_StartTime; static double m_MainTime; static bool m_ScriptDebuggerEnabled; + + struct LastReloadFailed + { #ifdef _WIN32 - static double m_LastReloadFailed; + typedef std::shared_lock SharedLock; + typedef std::unique_lock UniqueLock; + + std::shared_mutex Mutex; #else /* _WIN32 */ - typedef Atomic AtomicTs; - static_assert(AtomicTs::is_always_lock_free); - static SharedMemory m_LastReloadFailed; + typedef boost::interprocess::sharable_lock SharedLock; + typedef boost::interprocess::scoped_lock UniqueLock; + + boost::interprocess::interprocess_sharable_mutex Mutex; #endif /* _WIN32 */ + double When = 0; + char Why[16 * 1024] = {0}; + }; + + static LastReloadFailed* m_LastReloadFailed; + static LastReloadFailed* AllocLastReloadFailed(); #ifdef _WIN32 static BOOL WINAPI CtrlHandler(DWORD type); diff --git a/lib/base/logger.hpp b/lib/base/logger.hpp index 10e0872ae68..cdda3fa0eef 100644 --- a/lib/base/logger.hpp +++ b/lib/base/logger.hpp @@ -89,7 +89,6 @@ class Logger : public ObjectImpl void SetSeverity(const String& value, bool suppress_events = false, const Value& cookie = Empty) override; void ValidateSeverity(const Lazy& lvalue, const ValidationUtils& utils) final; -protected: void Start(bool runtimeCreated) override; void Stop(bool runtimeRemoved) override; diff --git a/lib/cli/daemoncommand.cpp b/lib/cli/daemoncommand.cpp index 3a9ce8c0a74..1782cdb2114 100644 --- a/lib/cli/daemoncommand.cpp +++ b/lib/cli/daemoncommand.cpp @@ -11,6 +11,7 @@ #include "base/atomic.hpp" #include "base/defer.hpp" #include "base/logger.hpp" +#include "base/streamlogger.hpp" #include "base/application.hpp" #include "base/process.hpp" #include "base/timer.hpp" @@ -25,6 +26,7 @@ #include #include #include +#include #ifdef _WIN32 #include @@ -222,6 +224,10 @@ static double GetDebugWorkerDelay() static String l_ObjectsPath; +#ifndef _WIN32 +static bool l_WorkerLoadedConfig = false; +#endif /* _WIN32 */ + /** * Do the actual work (config loading, ...) * @@ -246,6 +252,13 @@ int RunWorker(const std::vector& configs, bool closeConsoleLog = fa } #endif /* I2_DEBUG */ + std::ostringstream oss; + StreamLogger::Ptr sl = new StreamLogger(); + + sl->BindStream(&oss, false); + sl->Start(true); + sl->SetActive(true); + Log(LogInformation, "cli", "Loading configuration file(s)."); NotifyStatus("Loading configuration file(s)..."); @@ -255,14 +268,24 @@ int RunWorker(const std::vector& configs, bool closeConsoleLog = fa if (!DaemonUtility::LoadConfigFiles(configs, newItems, l_ObjectsPath, Configuration::VarsPath)) { Log(LogCritical, "cli", "Config validation failed. Re-run with 'icinga2 daemon -C' after fixing the config."); NotifyStatus("Config validation failed."); + + sl->Stop(true); + sl = nullptr; + Application::SetLastReloadFailed(Utility::GetTime(), oss.str()); + return EXIT_FAILURE; } + sl->Stop(true); + sl = nullptr; + oss = decltype(oss)(); + #ifndef _WIN32 Log(LogNotice, "cli") << "Notifying umbrella process (PID " << l_UmbrellaPid << ") about the config loading success"; (void)kill(l_UmbrellaPid, SIGUSR2); + l_WorkerLoadedConfig = true; Log(LogNotice, "cli") << "Waiting for the umbrella process to let us doing the actual work"; @@ -489,6 +512,7 @@ static pid_t StartUnixWorker(const std::vector& configs, bool close } (void)sigprocmask(SIG_UNBLOCK, &l_UnixWorkerSignals, nullptr); + Application::SetLastReloadFailed(Utility::GetTime(), "fork(2) failed"); return -1; case 0: @@ -531,6 +555,12 @@ static pid_t StartUnixWorker(const std::vector& configs, bool close } catch (const std::exception& ex) { Log(LogCritical, "cli") << "Failed to re-initialize thread pool after forking (child): " << DiagnosticInformation(ex); + + Application::SetLastReloadFailed( + Utility::GetTime(), + "Failed to re-initialize thread pool after forking (child): " + DiagnosticInformation(ex) + ); + _exit(EXIT_FAILURE); } @@ -539,14 +569,29 @@ static pid_t StartUnixWorker(const std::vector& configs, bool close } catch (const std::exception& ex) { Log(LogCritical, "cli") << "Failed to initialize process spawn helper after forking (child): " << DiagnosticInformation(ex); + + Application::SetLastReloadFailed( + Utility::GetTime(), + "Failed to initialize process spawn helper after forking (child): " + DiagnosticInformation(ex) + ); + _exit(EXIT_FAILURE); } _exit(RunWorker(configs, closeConsoleLog, stderrFile)); } catch (const std::exception& ex) { Log(LogCritical, "cli") << "Exception in main process: " << DiagnosticInformation(ex); + + if (!l_WorkerLoadedConfig) { + Application::SetLastReloadFailed(Utility::GetTime(), "Exception in main process: " + DiagnosticInformation(ex)); + } + _exit(EXIT_FAILURE); } catch (...) { + if (!l_WorkerLoadedConfig) { + Application::SetLastReloadFailed(Utility::GetTime(), "Exception in main process"); + } + _exit(EXIT_FAILURE); } @@ -813,7 +858,6 @@ int DaemonCommand::Run(const po::variables_map& vm, const std::vectorGetLastCheckResult(), &missingIcingaMinVersion, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros); + auto verbose (MacroProcessor::ResolveMacros("$icinga_verbose$", resolvers, checkable->GetLastCheckResult(), + nullptr, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros)); + if (resolvedMacros && !useResolvedMacros) return; @@ -158,10 +161,17 @@ void IcingaCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckRes ". Version: " + appVersion; /* Indicate a warning if the last reload failed. */ - double lastReloadFailed = Application::GetLastReloadFailed(); + auto lastReloadFailed (Application::GetLastReloadFailed()); + String verboseText; + + if (lastReloadFailed.first > 0) { + output += "; Last reload attempt failed at " + Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", lastReloadFailed.first); + + if (verbose.ToBool() && lastReloadFailed.second.GetLength()) { + output += ", see below"; + verboseText = lastReloadFailed.second; + } - if (lastReloadFailed > 0) { - output += "; Last reload attempt failed at " + Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", lastReloadFailed); state =ServiceWarning; } @@ -187,6 +197,10 @@ void IcingaCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckRes state = ServiceCritical; } + if (verboseText.GetLength()) { + output += "\n\n" + verboseText; + } + String commandName = command->GetName(); if (Checkable::ExecuteCommandProcessFinishedHandler) {