Support loading setting from config file and improve logging. (#118)

* support config load

* fix building error

* fix bugs

* commit again

* improve function params, std::string -> const std::string&

* refinement configure loading

* fixed code style

Co-authored-by: lgbo-ustc <[email protected]>

utils/local-engine/Builder/SerializedPlanBuilder.cpp

@@ -85,8 +85,7 @@ SchemaPtr SerializedSchemaBuilder::build()
     }
     return std::move(this->schema);
 }
-
-SerializedSchemaBuilder & SerializedSchemaBuilder::column(std::string name, std::string type, bool nullable)
+SerializedSchemaBuilder & SerializedSchemaBuilder::column(const std::string & name, const std::string & type, bool nullable)
 {
     this->type_map.emplace(name, type);
     this->nullability_map.emplace(name, nullable);
@@ -95,7 +94,7 @@ SerializedSchemaBuilder & SerializedSchemaBuilder::column(std::string name, std:
 SerializedSchemaBuilder::SerializedSchemaBuilder() : schema(new substrait::NamedStruct())
 {
 }
-SerializedPlanBuilder & SerializedPlanBuilder::registerFunction(int id, std::string name)
+SerializedPlanBuilder & SerializedPlanBuilder::registerFunction(int id, const std::string & name)
 {
     auto * extension = this->plan->mutable_extensions()->Add();
     auto * function_mapping = extension->mutable_extension_function();
@@ -139,7 +138,7 @@ SerializedPlanBuilder & SerializedPlanBuilder::filter(substrait::Expression * co
     return *this;
 }
 
-SerializedPlanBuilder & SerializedPlanBuilder::read(std::string path, SchemaPtr schema)
+SerializedPlanBuilder & SerializedPlanBuilder::read(const std::string & path, SchemaPtr schema)
 {
     substrait::Rel * rel = new substrait::Rel();
     auto * read = rel->mutable_read();
@@ -150,7 +149,13 @@ SerializedPlanBuilder & SerializedPlanBuilder::read(std::string path, SchemaPtr
     return *this;
 }
 
-SerializedPlanBuilder& SerializedPlanBuilder::readMergeTree(std::string database, std::string table, std::string relative_path,int min_block, int max_block, SchemaPtr schema)
+SerializedPlanBuilder & SerializedPlanBuilder::readMergeTree(
+    const std::string & database,
+    const std::string & table,
+    const std::string & relative_path,
+    int min_block,
+    int max_block,
+    SchemaPtr schema)
 {
     substrait::Rel * rel = new substrait::Rel();
     auto * read = rel->mutable_read();
@@ -234,7 +239,7 @@ substrait::Expression * literal(int32_t value)
     return rel;
 }
 
-substrait::Expression * literal(std::string value)
+substrait::Expression * literal(const std::string & value)
 {
     substrait::Expression * rel = new substrait::Expression();
     auto * literal = rel->mutable_literal();

utils/local-engine/Builder/SerializedPlanBuilder.h

@@ -39,14 +39,18 @@ class SerializedPlanBuilder
             .registerFunction(EQUAL_TO, "equal");
         return *this;
     }
-
-    SerializedPlanBuilder & registerFunction(int id, std::string name);
-    SerializedPlanBuilder & filter(substrait::Expression * condition);
-    SerializedPlanBuilder & project(std::vector<substrait::Expression *> projections);
-    SerializedPlanBuilder & aggregate(std::vector<int32_t> keys, std::vector<substrait::AggregateRel_Measure *> aggregates);
-    SerializedPlanBuilder & read(std::string path, SchemaPtr schema);
-    SerializedPlanBuilder &
-    readMergeTree(std::string database, std::string table, std::string relative_path, int min_block, int max_block, SchemaPtr schema);
+    SerializedPlanBuilder& registerFunction(int id, const std::string & name);
+    SerializedPlanBuilder& filter(substrait::Expression* condition);
+    SerializedPlanBuilder& project(std::vector<substrait::Expression*> projections);
+    SerializedPlanBuilder& aggregate(std::vector<int32_t> keys, std::vector<substrait::AggregateRel_Measure *> aggregates);
+    SerializedPlanBuilder& read(const std::string & path, SchemaPtr schema);
+    SerializedPlanBuilder & readMergeTree(
+        const std::string & database,
+        const std::string & table,
+        const std::string & relative_path,
+        int min_block,
+        int max_block,
+        SchemaPtr schema);
     std::unique_ptr<substrait::Plan> build();
 
 private:
@@ -68,8 +72,7 @@ class SerializedSchemaBuilder
 public:
     SerializedSchemaBuilder();
     SchemaPtr build();
-    SerializedSchemaBuilder & column(std::string name, std::string type, bool nullable = false);
-
+    SerializedSchemaBuilder& column(const std::string & name, const std::string & type, bool nullable = false);
 private:
     std::map<std::string, std::string> type_map;
     std::map<std::string, bool> nullability_map;
@@ -83,11 +86,10 @@ using MeasureList = std::vector<substrait::AggregateRel_Measure *>;
 substrait::Expression * scalarFunction(int32_t id, ExpressionList args);
 substrait::AggregateRel_Measure * measureFunction(int32_t id, ExpressionList args);
 
-substrait::Expression * literal(double_t value);
-substrait::Expression * literal(int32_t value);
-substrait::Expression * literal(std::string value);
-substrait::Expression * literalDate(int32_t value);
-substrait::Expression * literalTimestamp(int64_t value);
+substrait::Expression* literal(double_t value);
+substrait::Expression* literal(int32_t value);
+substrait::Expression* literal(const std::string & value);
+substrait::Expression* literalDate(int32_t value);
 
 substrait::Expression * selection(int32_t field_id);
 

utils/local-engine/CMakeLists.txt

@@ -122,6 +122,9 @@ target_compile_options(clickhouse_functions PRIVATE -fPIC)
 target_compile_options(clickhouse_common_access PRIVATE -fPIC)
 target_compile_options(clickhouse_storages_system PRIVATE -fPIC)
 target_compile_options(clickhouse_table_functions PRIVATE -fPIC)
+target_compile_options(substrait PRIVATE -fPIC)
+target_compile_options(loggers PRIVATE -fPIC)
+
 if (ENABLE_EMBEDDED_COMPILER)
     target_compile_options(LLVMDemangle PRIVATE -fPIC)
     target_compile_options(LLVMSupport PRIVATE -fPIC)

utils/local-engine/Common/JoinHelper.cpp

@@ -7,7 +7,7 @@ using namespace DB;
 namespace local_engine
 {
 
-JoinOptimizationInfo parseJoinOptimizationInfo(std::string optimization)
+JoinOptimizationInfo parseJoinOptimizationInfo(const std::string & optimization)
 {
     JoinOptimizationInfo info;
     ReadBufferFromString in(optimization);

utils/local-engine/Common/JoinHelper.h

@@ -11,7 +11,7 @@ struct JoinOptimizationInfo
 };
 
 
-JoinOptimizationInfo parseJoinOptimizationInfo(std::string optimization);
+JoinOptimizationInfo parseJoinOptimizationInfo(const std::string & optimization);
 
 
 }

utils/local-engine/Common/Logger.cpp

@@ -1,19 +1,26 @@
 #include "Logger.h"
+
+#include <loggers/Loggers.h>
 #include <Poco/ConsoleChannel.h>
 #include <Poco/AutoPtr.h>
 #include <Poco/AsyncChannel.h>
+#include <Poco/SimpleFileChannel.h>
 
 
 using Poco::ConsoleChannel;
 using Poco::AutoPtr;
 using Poco::AsyncChannel;
 
-void local_engine::Logger::initConsoleLogger()
+void local_engine::Logger::initConsoleLogger(const std::string & level)
 {
-    AutoPtr<ConsoleChannel> pCons(new ConsoleChannel);
-    AutoPtr<AsyncChannel> pAsync(new AsyncChannel(pCons));
-    Poco::Logger::root().setChannel(pAsync);
-    Poco::Logger::root().setLevel("error");
-    Poco::Logger::root().debug("init logger success");
+    AutoPtr<ConsoleChannel> chan(new ConsoleChannel);
+    AutoPtr<AsyncChannel> async_chann(new AsyncChannel(chan));
+    Poco::Logger::root().setChannel(async_chann);
+    Poco::Logger::root().setLevel(level);
 }
 
+void local_engine::Logger::initFileLogger(Poco::Util::AbstractConfiguration & config, const std::string & cmd_name)
+{
+    static Loggers loggers;
+    loggers.buildLoggers(config, Poco::Logger::root(), cmd_name);
+}

utils/local-engine/Common/Logger.h

@@ -1,13 +1,15 @@
 #pragma once
 
 #include <Poco/Logger.h>
+#include <Poco/Util/AbstractConfiguration.h>
 
 namespace local_engine
 {
 class Logger
 {
 public:
-    static void initConsoleLogger();
+    static void initConsoleLogger(const std::string & level);
+    static void initFileLogger(Poco::Util::AbstractConfiguration & config, const std::string & cmd_name);
 };
 }
 

utils/local-engine/Common/MergeTreeTool.cpp

@@ -42,7 +42,7 @@ std::unique_ptr<SelectQueryInfo> buildQueryInfo(NamesAndTypesList& names_and_typ
 }
 
 
-MergeTreeTable parseMergeTreeTableString(std::string & info)
+MergeTreeTable parseMergeTreeTableString(const std::string & info)
 {
     ReadBufferFromString in(info);
     assertString("MergeTree;", in);

utils/local-engine/Common/MergeTreeTool.h

@@ -34,5 +34,6 @@ struct MergeTreeTable
     std::string toString() const;
 };
 
-MergeTreeTable parseMergeTreeTableString(std::string & info);
+MergeTreeTable parseMergeTreeTableString(const std::string & info);
+
 }

utils/local-engine/Common/StringUtils.cpp

@@ -4,7 +4,7 @@
 
 namespace local_engine
 {
-PartitionValues StringUtils::parsePartitionTablePath(std::string file)
+PartitionValues StringUtils::parsePartitionTablePath(const std::string & file)
 {
     PartitionValues result;
     Poco::StringTokenizer path(file, "/");
@@ -18,7 +18,7 @@ PartitionValues StringUtils::parsePartitionTablePath(std::string file)
     }
     return result;
 }
-bool StringUtils::isNullPartitionValue(std::string value)
+bool StringUtils::isNullPartitionValue(const std::string & value)
 {
     return value == "__HIVE_DEFAULT_PARTITION__";
 }

utils/local-engine/Common/StringUtils.h

@@ -10,7 +10,7 @@ using PartitionValues = std::vector<PartitionValue>;
 class StringUtils
 {
 public:
-    static PartitionValues parsePartitionTablePath(std::string file);
-    static bool isNullPartitionValue(std::string value);
+    static PartitionValues parsePartitionTablePath(const std::string & file);
+    static bool isNullPartitionValue(const std::string & value);
 };
 }