Skip to content

Commit

Permalink
Merge pull request #173 from GoogleCloudPlatform/5789B3FF9E750C15D6B7…
Browse files Browse the repository at this point in the history 
…4524C048BD98

Project import generated by Copybara.
  • Loading branch information
@olavloite
olavloite authored Jun 7, 2024
2 parents d1ec59f + 7d2b556 commit 661769a
Show file tree
Hide file tree
Showing 139 changed files with 4,702 additions and 616 deletions.
22 changes: 21 additions & 1 deletion backend/actions/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,13 +105,22 @@ cc_library(
hdrs = ["check_constraint.h"],
deps = [
":action",
":context",
":ops",
"//backend/datamodel:key",
"//backend/datamodel:key_range",
"//backend/query:analyzer_options",
"//backend/schema/catalog:schema",
"//backend/storage:iterator",
"//common:errors",
"@com_google_absl//absl/log:check",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/types:span",
"@com_google_zetasql//zetasql/base:ret_check",
"@com_google_zetasql//zetasql/public:analyzer_options",
"@com_google_zetasql//zetasql/public:catalog",
"@com_google_zetasql//zetasql/public:evaluator",
"@com_google_zetasql//zetasql/public:type",
"@com_google_zetasql//zetasql/public:value",
],
)
Expand Down Expand Up @@ -217,6 +226,7 @@ cc_library(
"//third_party/spanner_pg/datatypes/extended:spanner_extended_type",
"@com_github_googleapis_google_cloud_cpp//:spanner",
"@com_google_absl//absl/container:flat_hash_map",
"@com_google_absl//absl/container:flat_hash_set",
"@com_google_absl//absl/log",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
Expand Down Expand Up @@ -252,15 +262,25 @@ cc_library(
hdrs = ["generated_column.h"],
deps = [
":action",
":context",
":ops",
"//backend/access:write",
"//backend/common:graph_dependency_helper",
"//backend/common:ids",
"//backend/datamodel:key",
"//backend/datamodel:key_range",
"//backend/query:analyzer_options",
"//backend/schema/catalog:schema",
"//backend/storage:iterator",
"//common:errors",
"@com_google_absl//absl/container:flat_hash_map",
"@com_google_absl//absl/container:flat_hash_set",
"@com_google_absl//absl/log:check",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings",
"@com_google_zetasql//zetasql/base:ret_check",
"@com_google_zetasql//zetasql/public:analyzer_options",
"@com_google_zetasql//zetasql/public:catalog",
"@com_google_zetasql//zetasql/public:evaluator",
"@com_google_zetasql//zetasql/public:value",
Expand Down
4 changes: 0 additions & 4 deletions backend/actions/OWNERS
View file

This file was deleted.

256 changes: 159 additions & 97 deletions backend/actions/change_stream.cc
View file

Large diffs are not rendered by default.

21 changes: 17 additions & 4 deletions backend/actions/check_constraint.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,25 @@
#include <utility>
#include <vector>

#include "zetasql/public/analyzer_options.h"
#include "zetasql/public/catalog.h"
#include "zetasql/public/evaluator.h"
#include "zetasql/public/types/type_factory.h"
#include "zetasql/public/value.h"
#include "absl/log/check.h"
#include "absl/status/status.h"
#include "absl/strings/substitute.h"
#include "absl/types/span.h"
#include "backend/actions/context.h"
#include "backend/actions/ops.h"
#include "backend/datamodel/key.h"
#include "backend/datamodel/key_range.h"
#include "backend/query/analyzer_options.h"
#include "backend/schema/catalog/check_constraint.h"
#include "backend/schema/catalog/column.h"
#include "backend/schema/catalog/table.h"
#include "backend/storage/iterator.h"
#include "common/errors.h"
#include "zetasql/base/ret_check.h"
#include "zetasql/base/status_macros.h"

namespace google {
Expand Down Expand Up @@ -77,16 +87,19 @@ absl::Status CheckConstraintVerifier::VerifyInsertUpdateOp(
const ActionContext* ctx, const Table* table,
const std::vector<const Column*>& columns,
const std::vector<zetasql::Value>& values, const Key& key) const {
absl::Span<const Column* const> dependent_columns =
check_constraint_->dependent_columns();
ZETASQL_ASSIGN_OR_RETURN(
std::unique_ptr<StorageIterator> itr,
ctx->store()->Read(table, KeyRange::Point(key), table->columns()));
ctx->store()->Read(table, KeyRange::Point(key), dependent_columns));
ZETASQL_RET_CHECK(itr->Next());
ZETASQL_RETURN_IF_ERROR(itr->Status());
ZETASQL_RET_CHECK_EQ(columns.size(), values.size());
ZETASQL_RET_CHECK_EQ(dependent_columns.size(), itr->NumColumns());

zetasql::ParameterValueMap column_values;
for (int i = 0; i < itr->NumColumns(); ++i) {
column_values[table->columns().at(i)->Name()] = itr->ColumnValue(i);
for (int i = 0; i < dependent_columns.size(); ++i) {
column_values[dependent_columns[i]->Name()] = itr->ColumnValue(i);
}
// The generated column values have been updated in
// GeneratedColumnEffector::Effect before the CheckConstraintVerifier is
Expand Down
23 changes: 23 additions & 0 deletions backend/actions/column_value.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

#include "backend/actions/column_value.h"

#include <cstdint>
#include <vector>

#include "zetasql/public/functions/string.h"
Expand Down Expand Up @@ -109,6 +110,18 @@ absl::Status ValidateColumnBytesValue(const Table* table, const Column* column,

absl::Status ValidateColumnArrayValue(const Table* table, const Column* column,
const zetasql::Value& value) {
// Validate the vector search array length.
if (column->has_vector_length() && !value.is_null()) {
int array_length = value.elements().size();
int expected_length = column->vector_length().value();
if (array_length > expected_length) {
return error::VectorLengthExceedsLimit(column->FullName(), array_length,
expected_length);
} else if (array_length < expected_length) {
return error::VectorLengthLessThanLimit(column->FullName(), array_length,
expected_length);
}
}
// Validate that bytes and string array element types do not exceed max
// length.
if (!value.is_null()) {
Expand All @@ -121,6 +134,16 @@ absl::Status ValidateColumnArrayValue(const Table* table, const Column* column,
ZETASQL_RETURN_IF_ERROR(ValidateColumnBytesValue(table, column, element));
}
}
// Validate no element in vector search array is null.
// Currently vector length can only be applied on ARRAY containing FLOAT32
// or FLOAT64 elements.
if (column->has_vector_length()) {
for (const auto& element : value.elements()) {
if (element.is_null()) {
return error::DisallowNullsInSearchArray(column->FullName());
}
}
}
}
return absl::OkStatus();
}
Expand Down
22 changes: 22 additions & 0 deletions backend/actions/column_value_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,14 @@ using zetasql::values::Bytes;
using zetasql::values::BytesArray;
using zetasql::values::Date;
using zetasql::values::Double;
using zetasql::values::Float;
using zetasql::values::Int64;
using zetasql::values::Json;
using zetasql::values::NullBool;
using zetasql::values::NullBytes;
using zetasql::values::NullDate;
using zetasql::values::NullDouble;
using zetasql::values::NullFloat;
using zetasql::values::NullInt64;
using zetasql::values::NullJson;
using zetasql::values::NullNumeric;
Expand All @@ -77,6 +79,7 @@ struct Values {
zetasql::Value int64_col = Int64(1);
zetasql::Value bool_col = Bool(true);
zetasql::Value date_col = Date(2);
zetasql::Value float_col = Float(2.0);
zetasql::Value double_col = Double(2.0);
zetasql::Value string_col = String("test");
zetasql::Value bytes_col = Bytes("01234");
Expand All @@ -101,6 +104,7 @@ class ColumnValueTest : public test::ActionsTest {
"int64_col INT64 NOT NULL",
"bool_col BOOL NOT NULL",
"date_col DATE NOT NULL",
"float_col FLOAT32 NOT NULL",
"double_col FLOAT64 NOT NULL",
"string_col STRING(MAX) NOT NULL",
"bytes_col BYTES(MAX) NOT NULL",
Expand Down Expand Up @@ -128,6 +132,7 @@ class ColumnValueTest : public test::ActionsTest {
ctx(),
Insert(table_, Key({Int64(1)}), base_columns_,
{values.int64_col, values.bool_col, values.date_col,
values.float_col,
values.double_col, values.string_col, values.bytes_col,
values.timestamp_col, values.array_string_col,
values.array_bytes_col, values.numeric_col, values.json_col}));
Expand All @@ -138,6 +143,7 @@ class ColumnValueTest : public test::ActionsTest {
ctx(),
Update(table_, Key({Int64(1)}), base_columns_,
{values.int64_col, values.bool_col, values.date_col,
values.float_col,
values.double_col, values.string_col, values.bytes_col,
values.timestamp_col, values.array_string_col,
values.array_bytes_col, values.numeric_col, values.json_col}));
Expand Down Expand Up @@ -181,6 +187,14 @@ TEST_F(ColumnValueTest, ValidateNotNullColumns) {
EXPECT_THAT(ValidateUpdate(values),
StatusIs(absl::StatusCode::kFailedPrecondition));
}
{
Values values;
values.float_col = NullFloat();
EXPECT_THAT(ValidateInsert(values),
StatusIs(absl::StatusCode::kFailedPrecondition));
EXPECT_THAT(ValidateUpdate(values),
StatusIs(absl::StatusCode::kFailedPrecondition));
}
{
Values values;
values.double_col = NullDouble();
Expand Down Expand Up @@ -258,6 +272,14 @@ TEST_F(ColumnValueTest, ValidateColumnsAreCorrectTypes) {
EXPECT_THAT(ValidateUpdate(values),
StatusIs(absl::StatusCode::kFailedPrecondition));
}
{
Values values;
values.float_col = String("");
EXPECT_THAT(ValidateInsert(values),
StatusIs(absl::StatusCode::kFailedPrecondition));
EXPECT_THAT(ValidateUpdate(values),
StatusIs(absl::StatusCode::kFailedPrecondition));
}
{
Values values;
values.double_col = String("");
Expand Down
76 changes: 59 additions & 17 deletions backend/actions/generated_column.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,21 +17,36 @@
#include "backend/actions/generated_column.h"

#include <algorithm>
#include <cstddef>
#include <map>
#include <memory>
#include <string>
#include <utility>
#include <vector>

#include "zetasql/public/analyzer_options.h"
#include "zetasql/public/catalog.h"
#include "zetasql/public/evaluator.h"
#include "zetasql/public/value.h"
#include "absl/container/flat_hash_set.h"
#include "absl/log/check.h"
#include "absl/status/status.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_join.h"
#include "absl/status/statusor.h"
#include "absl/strings/string_view.h"
#include "absl/strings/substitute.h"
#include "backend/access/write.h"
#include "backend/actions/context.h"
#include "backend/actions/ops.h"
#include "backend/common/graph_dependency_helper.h"
#include "backend/common/ids.h"
#include "backend/datamodel/key.h"
#include "backend/datamodel/key_range.h"
#include "backend/query/analyzer_options.h"
#include "backend/schema/catalog/column.h"
#include "backend/schema/catalog/table.h"
#include "backend/storage/iterator.h"
#include "common/errors.h"
#include "zetasql/base/ret_check.h"
#include "zetasql/base/status_macros.h"

namespace google {
Expand All @@ -44,6 +59,10 @@ absl::string_view GetColumnName(const Column* const& column) {
return column->Name();
}

bool IsKeyColumn(const Column* column) {
return column->table()->FindKeyColumn(column->Name()) != nullptr;
}

absl::Status GetGeneratedColumnsInTopologicalOrder(
const Table* table, std::vector<const Column*>* generated_columns) {
GraphDependencyHelper<const Column*, GetColumnName> sorter(
Expand Down Expand Up @@ -73,8 +92,7 @@ bool IsAnyDependentColumnPresent(
std::vector<std::string> user_supplied_columns) {
ABSL_DCHECK(generated_column->is_generated());
for (const auto& dep_col : generated_column->dependent_columns()) {
if (dep_col->is_generated() &&
generated_column->table()->FindKeyColumn(dep_col->Name()) != nullptr) {
if (dep_col->is_generated() && IsKeyColumn(dep_col)) {
return IsAnyDependentColumnPresent(dep_col, user_supplied_columns);
}
if (std::find(user_supplied_columns.begin(), user_supplied_columns.end(),
Expand Down Expand Up @@ -116,11 +134,17 @@ absl::Status GeneratedColumnEffector::Initialize(
zetasql::Catalog* function_catalog) {
ZETASQL_RETURN_IF_ERROR(
GetGeneratedColumnsInTopologicalOrder(table_, &generated_columns_));
absl::flat_hash_set<ColumnID> unique_dependent_column;
expressions_.reserve(generated_columns_.size());
for (const Column* generated_column : generated_columns_) {
ZETASQL_ASSIGN_OR_RETURN(auto expr,
PrepareExpression(generated_column, function_catalog));
expressions_[generated_column] = std::move(expr);
for (const Column* dep : generated_column->dependent_columns()) {
if (unique_dependent_column.insert(dep->id()).second) {
dependent_columns_.push_back(dep);
}
}
}
return absl::OkStatus();
}
Expand Down Expand Up @@ -159,7 +183,7 @@ absl::Status GeneratedColumnEffector::Effect(
for (int i = 0; i < generated_columns_.size(); ++i) {
const Column* generated_column = generated_columns_[i];
if (!generated_column->has_default_value() &&
table_->FindKeyColumn(generated_column->Name()) == nullptr) {
!IsKeyColumn(generated_column)) {
// skip non-key columns except default columns since generated key columns
// may depend on default columns values of which would need to be
// evaluated.
Expand Down Expand Up @@ -242,35 +266,43 @@ absl::Status GeneratedColumnEffector::Effect(const ActionContext* ctx,
}
}

return Effect(ctx, op.key, &column_values);
return Effect(ctx, op.key, &column_values, /*skip_default_values=*/false);
}

absl::Status GeneratedColumnEffector::Effect(const ActionContext* ctx,
const UpdateOp& op) const {
if (!op.columns.empty() && op.columns.at(0)->is_generated()) {
// This is a generated column effect. Don't process it again.
return absl::OkStatus();
for (const Column* column : op.columns) {
// If any non-key generated columns appear then this is a generated column
// effect and we do not need to process it again. The non-key requirement is
// needed because user-generated updates are expected to include key values,
// including generated ones.
if (column->is_generated() && !IsKeyColumn(column)) {
return absl::OkStatus();
}
}

zetasql::ParameterValueMap column_values;
ZETASQL_ASSIGN_OR_RETURN(
std::unique_ptr<StorageIterator> itr,
ctx->store()->Read(table_, KeyRange::Point(op.key), table_->columns()));
ctx->store()->Read(table_, KeyRange::Point(op.key), dependent_columns_));
ZETASQL_RET_CHECK(itr->Next());
ZETASQL_RETURN_IF_ERROR(itr->Status());
ZETASQL_RET_CHECK_EQ(op.columns.size(), op.values.size());
for (int i = 0; i < itr->NumColumns(); ++i) {
column_values[table_->columns().at(i)->Name()] = itr->ColumnValue(i);
ZETASQL_RET_CHECK_EQ(itr->NumColumns(), dependent_columns_.size());
for (int i = 0; i < dependent_columns_.size(); ++i) {
column_values[dependent_columns_[i]->Name()] = itr->ColumnValue(i);
}
for (int i = 0; i < op.columns.size(); ++i) {
column_values[op.columns[i]->Name()] = op.values[i];
}
return Effect(ctx, op.key, &column_values);
return Effect(ctx, op.key, &column_values, /*skip_default_values=*/true);
}

absl::Status GeneratedColumnEffector::Effect(
const ActionContext* ctx, const Key& key,
zetasql::ParameterValueMap* column_values) const {
zetasql::ParameterValueMap* column_values,
bool skip_default_values) const {
ZETASQL_RET_CHECK(for_keys_ == false);
std::vector<zetasql::Value> generated_values;
generated_values.reserve(generated_columns_.size());

Expand All @@ -280,10 +312,20 @@ absl::Status GeneratedColumnEffector::Effect(
// Evaluate generated columns in topological order.
for (int i = 0; i < generated_columns_.size(); ++i) {
const Column* generated_column = generated_columns_[i];
// If this column has a default value and the user is supplying a value
// for it, then we don't need to compute its default value.
// Default values should only be populated for inserts where no value is
// supplied by the user. For updates, skip_default_values is set to true
// and all default value computations are skipped (the column will thus
// either be set to a new user-provided value or left at its current value).
if (generated_column->has_default_value() &&
column_values->find(generated_column->Name()) != column_values->end()) {
(skip_default_values || column_values->find(generated_column->Name()) !=
column_values->end())) {
continue;
}
// Keys are handled by a separate effector initialized with for_keys_=true.
// Skipping these columns here is not simply an optimization; if we include
// them then we may find ourselves in an infinite loop repeatedly generating
// effects for generated PKs.
if (IsKeyColumn(generated_column)) {
continue;
}

Expand Down
Loading

0 comments on commit 661769a

Please sign in to comment.