[YQL-19309] Switch YT provider to DQ nodes from yql/essentials (#12158)

ydb-platform · Dec 1, 2024 · 5f44598 · 5f44598
1 parent 42575b9
commit 5f44598
Show file tree

Hide file tree

Showing 12 changed files with 33 additions and 27 deletions.
diff --git a/ydb/library/yql/providers/yt/gateway/file/yql_yt_file_mkql_compiler.cpp b/ydb/library/yql/providers/yt/gateway/file/yql_yt_file_mkql_compiler.cpp
@@ -1,14 +1,15 @@
 #include "yql_yt_file_mkql_compiler.h"
 
-#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
 #include <ydb/library/yql/providers/yt/expr_nodes/yql_yt_expr_nodes.h>
 #include <ydb/library/yql/providers/yt/provider/yql_yt_table.h>
 #include <ydb/library/yql/providers/yt/provider/yql_yt_helpers.h>
 #include <ydb/library/yql/providers/yt/provider/yql_yt_mkql_compiler.h>
-#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
+
 #include <yql/essentials/providers/result/expr_nodes/yql_res_expr_nodes.h>
 #include <yql/essentials/providers/common/mkql/yql_type_mkql.h>
 #include <yql/essentials/providers/common/schema/mkql/yql_mkql_schema.h>
+#include <yql/essentials/core/dq_expr_nodes/dq_expr_nodes.h>
+#include <yql/essentials/core/dqs_expr_nodes/dqs_expr_nodes.h>
 #include <yql/essentials/core/expr_nodes/yql_expr_nodes.h>
 #include <yql/essentials/core/yql_opt_utils.h>
 #include <yql/essentials/utils/yql_panic.h>
@@ -31,6 +32,7 @@ namespace NYql {
 using namespace NKikimr;
 using namespace NKikimr::NMiniKQL;
 using namespace NNodes;
+using namespace NNodes::NDq;
 
 namespace {
 

diff --git a/ydb/library/yql/providers/yt/gateway/native/ya.make b/ydb/library/yql/providers/yt/gateway/native/ya.make
@@ -40,8 +40,6 @@ PEERDIR(
     yql/essentials/providers/common/provider
     yql/essentials/providers/common/schema/expr
     yql/essentials/providers/result/expr_nodes
-    ydb/library/yql/providers/stat/expr_nodes
-    ydb/library/yql/providers/stat/uploader
     ydb/library/yql/providers/yt/codec
     ydb/library/yql/providers/yt/common
     ydb/library/yql/providers/yt/expr_nodes

diff --git a/ydb/library/yql/providers/yt/gateway/native/yql_yt_native.cpp b/ydb/library/yql/providers/yt/gateway/native/yql_yt_native.cpp
@@ -33,14 +33,11 @@
 #include <yql/essentials/providers/common/schema/expr/yql_expr_schema.h>
 #include <yql/essentials/providers/common/proto/gateways_config.pb.h>
 #include <yql/essentials/providers/result/expr_nodes/yql_res_expr_nodes.h>
-#include <ydb/library/yql/providers/stat/expr_nodes/yql_stat_expr_nodes.h>
-#include <ydb/library/yql/providers/stat/uploader/yql_stat_uploader.h>
 
 #include <yql/essentials/ast/yql_expr.h>
 #include <yql/essentials/core/issue/yql_issue.h>
 #include <yql/essentials/core/yql_type_helpers.h>
 #include <yql/essentials/core/yql_graph_transformer.h>
-#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
 #include <yql/essentials/utils/log/log.h>
 #include <yql/essentials/minikql/mkql_node.h>
 #include <yql/essentials/minikql/mkql_node_cast.h>

diff --git a/ydb/library/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_key_range.cpp b/ydb/library/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_key_range.cpp
@@ -2,13 +2,14 @@
 #include "yql_yt_phy_opt_helper.h"
 
 #include <ydb/library/yql/providers/yt/provider/yql_yt_helpers.h>
-#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
 
+#include <yql/essentials/core/dq_expr_nodes/dq_expr_nodes.h>
 #include <yql/essentials/utils/log/log.h>
 
 namespace NYql {
 
 using namespace NNodes;
+using namespace NNodes::NDq;
 using namespace NPrivate;
 
 // All keyFilter settings are combined by OR.

diff --git a/ydb/library/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_write.cpp b/ydb/library/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_write.cpp
@@ -5,17 +5,17 @@
 #include <ydb/library/yql/providers/yt/provider/yql_yt_optimize.h>
 #include <ydb/library/yql/providers/yt/opt/yql_yt_key_selector.h>
 #include <ydb/library/yql/providers/stat/expr_nodes/yql_stat_expr_nodes.h>
-#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
-#include <yql/essentials/providers/common/codec/yql_codec_type_flags.h>
-
-#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
 
+#include <yql/essentials/core/dq_expr_nodes/dq_expr_nodes.h>
+#include <yql/essentials/core/dqs_expr_nodes/dqs_expr_nodes.h>
 #include <yql/essentials/core/yql_opt_utils.h>
 #include <yql/essentials/core/yql_type_helpers.h>
+#include <yql/essentials/providers/common/codec/yql_codec_type_flags.h>
 
 namespace NYql {
 
 using namespace NNodes;
+using namespace NNodes::NDq;
 using namespace NPrivate;
 
 TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::DqWrite(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) const {

diff --git a/ydb/library/yql/providers/yt/provider/ya.make b/ydb/library/yql/providers/yt/provider/ya.make
@@ -77,6 +77,8 @@ PEERDIR(
     yql/essentials/utils
     yql/essentials/utils/log
     yql/essentials/core
+    yql/essentials/core/dq_expr_nodes
+    yql/essentials/core/dqs_expr_nodes
     yql/essentials/core/expr_nodes
     yql/essentials/core/issue
     yql/essentials/core/issue/protos
@@ -96,8 +98,6 @@ PEERDIR(
     yql/essentials/providers/common/schema/expr
     yql/essentials/providers/common/structured_token
     yql/essentials/providers/common/transform
-    ydb/library/yql/dq/expr_nodes
-    ydb/library/yql/providers/dq/expr_nodes
     yql/essentials/providers/result/expr_nodes
     ydb/library/yql/providers/stat/expr_nodes
     ydb/library/yql/providers/yt/common

diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_dq_hybrid.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_dq_hybrid.cpp
@@ -9,14 +9,14 @@
 #include <yql/essentials/providers/common/provider/yql_provider_names.h>
 #include <yql/essentials/providers/common/provider/yql_provider.h>
 #include <yql/essentials/providers/common/transform/yql_optimize.h>
-#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
+#include <yql/essentials/core/dq_expr_nodes/dq_expr_nodes.h>
+#include <yql/essentials/core/dqs_expr_nodes/dqs_expr_nodes.h>
 #include <yql/essentials/core/expr_nodes/yql_expr_nodes.h>
 #include <yql/essentials/core/yql_expr_type_annotation.h>
 #include <yql/essentials/core/yql_expr_optimize.h>
 #include <yql/essentials/core/yql_opt_utils.h>
 #include <yql/essentials/core/yql_type_helpers.h>
 #include <yql/essentials/core/yql_data_provider.h>
-#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
 #include <yql/essentials/utils/log/log.h>
 #include <yql/essentials/utils/yql_panic.h>
 #include <yql/essentials/minikql/mkql_program_builder.h>
@@ -31,7 +31,7 @@ namespace NYql {
 namespace {
 
 using namespace NNodes;
-using namespace NDq;
+using namespace NNodes::NDq;
 
 class TYtDqHybridTransformer : public TOptimizeTransformerBase {
 public:

diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_dq_integration.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_dq_integration.cpp
@@ -13,10 +13,10 @@
 #include <yql/essentials/providers/common/dq/yql_dq_integration_impl.h>
 #include <yql/essentials/providers/common/codec/yql_codec_type_flags.h>
 #include <yql/essentials/providers/common/config/yql_dispatch.h>
-#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
-#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
 #include <yql/essentials/providers/result/expr_nodes/yql_res_expr_nodes.h>
 #include <yql/essentials/ast/yql_expr.h>
+#include <yql/essentials/core/dq_expr_nodes/dq_expr_nodes.h>
+#include <yql/essentials/core/dqs_expr_nodes/dqs_expr_nodes.h>
 #include <yql/essentials/core/yql_type_helpers.h>
 #include <yql/essentials/core/yql_expr_optimize.h>
 #include <yql/essentials/core/yql_opt_utils.h>
@@ -39,6 +39,7 @@ static const THashSet<TStringBuf> UNSUPPORTED_YT_PRAGMAS = {"maxrowweight",  "la
 static const THashSet<TStringBuf> POOL_TREES_WHITELIST = {"physical",  "cloud", "cloud_default"};
 
 using namespace NNodes;
+using namespace NNodes::NDq;
 
 bool CheckSupportedTypesOld(const TTypeAnnotationNode::TListType& typesToCheck, const TSet<TString>& supportedTypes, const TSet<NUdf::EDataSlot>& supportedDataTypes, std::function<void(const TString&)> unsupportedTypeHandler) {
     TSet<ETypeAnnotationKind> supported;

diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_dq_optimize.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_dq_optimize.cpp
@@ -3,15 +3,17 @@
 #include "yql_yt_optimize.h"
 
 #include <ydb/library/yql/providers/yt/expr_nodes/yql_yt_expr_nodes.h>
-#include <yql/essentials/providers/common/dq/yql_dq_optimization_impl.h>
-#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
+
+#include <yql/essentials/core/dqs_expr_nodes/dqs_expr_nodes.h>
 #include <yql/essentials/core/yql_expr_optimize.h>
+#include <yql/essentials/providers/common/dq/yql_dq_optimization_impl.h>
 #include <yql/essentials/utils/log/log.h>
 
 
 namespace NYql {
 
 using namespace NNodes;
+using namespace NNodes::NDq;
 
 class TYtDqOptimizers: public TDqOptimizationBase {
 public:

diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_helpers.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_helpers.cpp
@@ -4,14 +4,14 @@
 #include "yql_yt_op_hash.h"
 #include "yql_yt_optimize.h"
 
-#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
-#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
 #include <ydb/library/yql/providers/yt/lib/mkql_helpers/mkql_helpers.h>
 #include <ydb/library/yql/providers/yt/common/yql_configuration.h>
 #include <ydb/library/yql/providers/yt/opt/yql_yt_key_selector.h>
 #include <yql/essentials/providers/common/provider/yql_provider.h>
 #include <yql/essentials/providers/common/codec/yql_codec_type_flags.h>
 #include <yql/essentials/providers/common/codec/yql_codec.h>
+#include <yql/essentials/core/dq_expr_nodes/dq_expr_nodes.h>
+#include <yql/essentials/core/dqs_expr_nodes/dqs_expr_nodes.h>
 #include <yql/essentials/core/expr_nodes/yql_expr_nodes.h>
 #include <yql/essentials/core/type_ann/type_ann_expr.h>
 #include <yql/essentials/core/type_ann/type_ann_core.h>
@@ -37,6 +37,7 @@
 namespace NYql {
 
 using namespace NNodes;
+using namespace NNodes::NDq;
 
 namespace {
 

diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_mkql_compiler.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_mkql_compiler.cpp
@@ -6,14 +6,15 @@
 #include <ydb/library/yql/providers/yt/lib/row_spec/yql_row_spec.h>
 #include <ydb/library/yql/providers/yt/lib/skiff/yql_skiff_schema.h>
 #include <ydb/library/yql/providers/yt/lib/mkql_helpers/mkql_helpers.h>
-#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
-#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
+
+#include <yql/essentials/core/dq_expr_nodes/dq_expr_nodes.h>
+#include <yql/essentials/core/dqs_expr_nodes/dqs_expr_nodes.h>
+#include <yql/essentials/core/yql_opt_utils.h>
 #include <yql/essentials/providers/common/codec/yql_codec_type_flags.h>
 #include <yql/essentials/providers/common/mkql/yql_type_mkql.h>
 #include <yql/essentials/minikql/mkql_program_builder.h>
 #include <yql/essentials/minikql/mkql_node_cast.h>
 #include <yql/essentials/minikql/defs.h>
-#include <yql/essentials/core/yql_opt_utils.h>
 #include <yql/essentials/utils/log/context.h>
 
 #include <library/cpp/yson/node/node_io.h>
@@ -28,6 +29,7 @@ namespace NYql {
 using namespace NKikimr;
 using namespace NKikimr::NMiniKQL;
 using namespace NNodes;
+using namespace NNodes::NDq;
 
 TRuntimeNode BuildTableContentCall(TStringBuf callName,
     TType* outItemType,

diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_op_hash.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_op_hash.cpp
@@ -4,7 +4,8 @@
 
 #include <ydb/library/yql/providers/yt/expr_nodes/yql_yt_expr_nodes.h>
 #include <ydb/library/yql/providers/yt/lib/hash/yql_hash_builder.h>
-#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
+
+#include <yql/essentials/core/dq_expr_nodes/dq_expr_nodes.h>
 #include <yql/essentials/utils/log/log.h>
 
 #include <yql/essentials/utils/yql_panic.h>
@@ -17,6 +18,7 @@
 namespace NYql {
 
 using namespace NNodes;
+using namespace NNodes::NDq;
 
 TYtNodeHashCalculator::TYtNodeHashCalculator(const TYtState::TPtr& state, const TString& cluster, const TYtSettings::TConstPtr& config)
     : TNodeHashCalculator(*state->Types, state->NodeHash, MakeSalt(config, cluster))