Skip to content

Commit

Permalink
[GLUTEN-4545] [CH] Rename Common/Logger.h to avoid conflict. (#4546)
Browse files Browse the repository at this point in the history
* fix build due to #4457

(cherry picked from commit c91c18e)

* Rename Logger.h to LoggerExtend.h due to this PR(ClickHouse/ClickHouse#58831). It adds a Logger.h in the Common directory, which conflicts with our extern-local-engine/Common/Logger.h

* style
  • Loading branch information
baibaichen authored Jan 27, 2024
1 parent 15f51e8 commit 5db501a
Show file tree
Hide file tree
Showing 8 changed files with 30 additions and 593 deletions.
39 changes: 11 additions & 28 deletions cpp-ch/local-engine/Common/CHUtil.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,13 @@
#include <Core/Defines.h>
#include <Core/NamesAndTypes.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/DataTypeMap.h>
#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypeTuple.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/NestedUtils.h>
#include <Disks/registerDisks.h>
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionsConversion.h>
#include <Functions/registerFunctions.h>
Expand All @@ -49,6 +50,7 @@
#include <QueryPipeline/QueryPipelineBuilder.h>
#include <QueryPipeline/printPipeline.h>
#include <Storages/Output/WriteBufferBuilder.h>
#include <Storages/StorageMergeTreeFactory.h>
#include <Storages/SubstraitSource/ReadBufferBuilder.h>
#include <google/protobuf/util/json_util.h>
#include <google/protobuf/wrappers.pb.h>
Expand All @@ -58,25 +60,23 @@
#include <Common/Config/ConfigProcessor.h>
#include <Common/CurrentThread.h>
#include <Common/GlutenSignalHandler.h>
#include <Common/Logger.h>
#include <Common/LoggerExtend.h>
#include <Common/logger_useful.h>
#include <Common/typeid_cast.h>
#include <Disks/registerDisks.h>
#include <Storages/StorageMergeTreeFactory.h>

#include <boost/algorithm/string/case_conv.hpp>
#include <boost/algorithm/string/predicate.hpp>

#include "CHUtil.h"

#include <sys/resource.h>
#include <unistd.h>
#include <sys/resource.h>

namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
extern const int BAD_ARGUMENTS;
}
}

Expand Down Expand Up @@ -304,7 +304,7 @@ size_t PODArrayUtil::adjustMemoryEfficientSize(size_t n)
}
else
{
padded_n = rounded_n - padding_n;
padded_n = rounded_n - padding_n;
}
return padded_n;
}
Expand All @@ -326,9 +326,7 @@ std::string PlanUtil::explainPlan(DB::QueryPlan & plan)
std::vector<MergeTreeUtil::Path> MergeTreeUtil::getAllMergeTreeParts(const Path & storage_path)
{
if (!fs::exists(storage_path))
{
throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Invalid merge tree store path:{}", storage_path.string());
}

// TODO: May need to check the storage format version
std::vector<fs::path> res;
Expand All @@ -346,9 +344,7 @@ DB::NamesAndTypesList MergeTreeUtil::getSchemaFromMergeTreePart(const fs::path &
{
DB::NamesAndTypesList names_types_list;
if (!fs::exists(part_path))
{
throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Invalid merge tree store path:{}", part_path.string());
}
DB::ReadBufferFromFile readbuffer((part_path / "columns.txt").string());
names_types_list.readText(readbuffer);
return names_types_list;
Expand Down Expand Up @@ -388,9 +384,7 @@ std::optional<DB::ColumnWithTypeAndName> NestedColumnExtractHelper::extractColum
{
auto table_iter = nested_tables.find(column_name_prefix);
if (table_iter == nested_tables.end())
{
return {};
}

auto & nested_table = table_iter->second;
auto nested_names = DB::Nested::splitName(column_name_suffix);
Expand All @@ -412,9 +406,7 @@ std::optional<DB::ColumnWithTypeAndName> NestedColumnExtractHelper::extractColum

const auto * sub_col = findColumn(*nested_table, new_column_name_prefix);
if (!sub_col)
{
return {};
}

DB::ColumnsWithTypeAndName columns = {*sub_col};
DB::Block sub_block(columns);
Expand All @@ -431,9 +423,7 @@ const DB::ColumnWithTypeAndName * NestedColumnExtractHelper::findColumn(const DB
const auto & cols = in_block.getColumnsWithTypeAndName();
auto found = std::find_if(cols.begin(), cols.end(), [&](const auto & column) { return boost::iequals(column.name, name); });
if (found == cols.end())
{
return nullptr;
}
return &*found;
}

Expand Down Expand Up @@ -476,9 +466,7 @@ std::map<std::string, std::string> BackendInitializerUtil::getBackendConfMap(std
{
std::map<std::string, std::string> ch_backend_conf;
if (plan == nullptr)
{
return ch_backend_conf;
}

/// Parse backend configs from plan extensions
do
Expand Down Expand Up @@ -576,9 +564,9 @@ void BackendInitializerUtil::initLoggers(DB::Context::ConfigurationPtr config)
{
auto level = config->getString("logger.level", "warning");
if (config->has("logger.log"))
local_engine::Logger::initFileLogger(*config, "ClickHouseBackend");
local_engine::LoggerExtend::initFileLogger(*config, "ClickHouseBackend");
else
local_engine::Logger::initConsoleLogger(level);
local_engine::LoggerExtend::initConsoleLogger(level);

logger = &Poco::Logger::get("ClickHouseBackend");
}
Expand Down Expand Up @@ -675,9 +663,7 @@ void BackendInitializerUtil::initContexts(DB::Context::ConfigurationPtr config)
/// Make sure global_context and shared_context are constructed only once.
auto & shared_context = SerializedPlanParser::shared_context;
if (!shared_context.get())
{
shared_context = SharedContextHolder(Context::createShared());
}

auto & global_context = SerializedPlanParser::global_context;
if (!global_context)
Expand Down Expand Up @@ -856,9 +842,7 @@ UInt64 MemoryUtil::getCurrentMemoryUsage(size_t depth)
Int64 current_memory_usage = 0;
auto * current_mem_tracker = DB::CurrentThread::getMemoryTracker();
for (size_t i = 0; i < depth && current_mem_tracker; ++i)
{
current_mem_tracker = current_mem_tracker->getParent();
}
if (current_mem_tracker)
current_memory_usage = current_mem_tracker->get();
return current_memory_usage < 0 ? 0 : current_memory_usage;
Expand All @@ -867,12 +851,11 @@ UInt64 MemoryUtil::getCurrentMemoryUsage(size_t depth)
UInt64 MemoryUtil::getMemoryRSS()
{
long rss = 0L;
FILE* fp = NULL;
FILE * fp = NULL;
char buf[4096];
sprintf(buf, "/proc/%d/statm", getpid());
if ((fp = fopen(buf, "r")) == NULL) {
if ((fp = fopen(buf, "r")) == NULL)
return 0;
}
fscanf(fp, "%*s%ld", &rss);
fclose(fp);
return rss * sysconf(_SC_PAGESIZE);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,24 +14,24 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "Logger.h"

#include "LoggerExtend.h"
#include <Loggers/Loggers.h>
#include <Poco/AsyncChannel.h>
#include <Poco/AutoPtr.h>
#include <Poco/ConsoleChannel.h>
#include <Poco/PatternFormatter.h>
#include <Poco/FormattingChannel.h>
#include <Poco/PatternFormatter.h>
#include <Poco/SimpleFileChannel.h>


using Poco::AsyncChannel;
using Poco::AutoPtr;
using Poco::ConsoleChannel;
using Poco::PatternFormatter;
using Poco::FormattingChannel;
using Poco::PatternFormatter;

void local_engine::Logger::initConsoleLogger(const std::string & level)
namespace local_engine
{
void LoggerExtend::initConsoleLogger(const std::string & level)
{
AutoPtr<ConsoleChannel> chan(new ConsoleChannel);

Expand All @@ -46,8 +46,9 @@ void local_engine::Logger::initConsoleLogger(const std::string & level)
Poco::Logger::root().setLevel(level);
}

void local_engine::Logger::initFileLogger(Poco::Util::AbstractConfiguration & config, const std::string & cmd_name)
void LoggerExtend::initFileLogger(Poco::Util::AbstractConfiguration & config, const std::string & cmd_name)
{
static Loggers loggers;
loggers.buildLoggers(config, Poco::Logger::root(), cmd_name);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,11 @@
*/
#pragma once

#include <Poco/Logger.h>
#include <Poco/Util/AbstractConfiguration.h>

namespace local_engine
{
class Logger
class LoggerExtend
{
public:
static void initConsoleLogger(const std::string & level = "error");
Expand Down
19 changes: 9 additions & 10 deletions cpp-ch/local-engine/examples/signal_demo.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,29 +15,28 @@
* limitations under the License.
*/

#include <IO/WriteHelpers.h>
#include <base/sleep.h>
#include <Common/GlutenSignalHandler.h>
#include <Common/Logger.h>
#include <Common/LoggerExtend.h>
#include <Common/logger_useful.h>
#include <IO/WriteBufferFromFileDescriptorDiscardOnFailure.h>
#include <IO/WriteHelpers.h>


using namespace DB;
using namespace local_engine;


int main(int /*argc*/, char * /*argv*/[])
int main(int /*argc*/, char * /*argv*/[])
{
local_engine::Logger::initConsoleLogger("trace");
local_engine::LoggerExtend::initConsoleLogger("trace");
Poco::Logger * logger = &Poco::Logger::get("signal_demo");
SignalHandler::instance().init();

for (int j = 0; j < 10 ; j++) {
for (int j = 0; j < 10; j++)
{
LOG_TRACE(logger, "counter {}", j);

if( j ){
int *x = nullptr;
if (j)
{
int * x = nullptr;
*x = 1;
}
sleepForSeconds(3);
Expand Down
Loading

0 comments on commit 5db501a

Please sign in to comment.