diff --git a/changelog/19.0/19.0.7/changelog.md b/changelog/19.0/19.0.7/changelog.md new file mode 100644 index 00000000000..a5c9a693f9c --- /dev/null +++ b/changelog/19.0/19.0.7/changelog.md @@ -0,0 +1,50 @@ +# Changelog of Vitess v19.0.7 + +### Bug fixes +#### Backup and Restore + * [release-19.0] Fail fast when builtinbackup fails to restore a single file (#16856) [#16867](https://github.com/vitessio/vitess/pull/16867) +#### Query Serving + * Backport: Fix ACL checks for CTEs (#16642) [#16776](https://github.com/vitessio/vitess/pull/16776) + * [release-19.0] VTTablet: smartconnpool: notify all expired waiters (#16897) [#16901](https://github.com/vitessio/vitess/pull/16901) + * [release-19.0] fixes bugs around expression precedence and LIKE (#16934 & #16649) [#16945](https://github.com/vitessio/vitess/pull/16945) + * [release-19.0] bugfix: add HAVING columns inside derived tables (#16976) [#16978](https://github.com/vitessio/vitess/pull/16978) + * [release-19.0] bugfix: treat EXPLAIN like SELECT (#17054) [#17056](https://github.com/vitessio/vitess/pull/17056) + * [release-19.0] Delegate Column Availability Checks to MySQL for Single-Route Queries (#17077) [#17085](https://github.com/vitessio/vitess/pull/17085) + * Bugfix for Panic on Joined Queries with Non-Authoritative Tables in Vitess 19.0 [#17103](https://github.com/vitessio/vitess/pull/17103) +#### VTAdmin + * [release-19.0] VTAdmin: Fix serve-handler's path-to-regexp dep and add default schema refresh (#16778) [#16783](https://github.com/vitessio/vitess/pull/16783) +#### VTGate + * [release-19.0] Support passing filters to `discovery.NewHealthCheck(...)` (#16170) [#16871](https://github.com/vitessio/vitess/pull/16871) + * [release-19.0] Fix deadlock between health check and topology watcher (#16995) [#17008](https://github.com/vitessio/vitess/pull/17008) +#### VTTablet + * [release-19.0] Fix race in `replicationLagModule` of `go/vt/throttle` (#16078) [#16899](https://github.com/vitessio/vitess/pull/16899) +### CI/Build +#### Docker + * [release-19.0] Remove mysql57 from docker images [#16763](https://github.com/vitessio/vitess/pull/16763) +#### General + * [release-19.0] Upgrade Golang to 1.22.8 [#16895](https://github.com/vitessio/vitess/pull/16895) +### Dependencies +#### Java + * [release-19.0] Bump com.google.protobuf:protobuf-java from 3.24.3 to 3.25.5 in /java (#16809) [#16837](https://github.com/vitessio/vitess/pull/16837) + * [release-19.0] Bump commons-io:commons-io from 2.7 to 2.14.0 in /java (#16889) [#16930](https://github.com/vitessio/vitess/pull/16930) +#### VTAdmin + * [release-19.0] VTAdmin: Address security vuln in path-to-regexp node pkg (#16770) [#16772](https://github.com/vitessio/vitess/pull/16772) +### Enhancement +#### Online DDL + * [release-19.0] Improve Schema Engine's TablesWithSize80 query (#17066) [#17089](https://github.com/vitessio/vitess/pull/17089) +### Internal Cleanup +#### VTAdmin + * [release-19.0] VTAdmin: Upgrade deps to address security vulns (#16843) [#16846](https://github.com/vitessio/vitess/pull/16846) +### Regression +#### Backup and Restore + * [release-19.0] Fix unreachable errors when taking a backup (#17062) [#17110](https://github.com/vitessio/vitess/pull/17110) +#### Query Serving + * [release-19.0] fix: route engine to handle column truncation for execute after lookup (#16981) [#16984](https://github.com/vitessio/vitess/pull/16984) + * [release-19.0] Add support for `MultiEqual` opcode for lookup vindexes. (#16975) [#17039](https://github.com/vitessio/vitess/pull/17039) +### Release +#### General + * [release-19.0] Code Freeze for `v19.0.7` [#17148](https://github.com/vitessio/vitess/pull/17148) +### Testing +#### Cluster management + * [release-19.0] Flaky test fixes (#16940) [#16958](https://github.com/vitessio/vitess/pull/16958) + diff --git a/changelog/19.0/19.0.7/release_notes.md b/changelog/19.0/19.0.7/release_notes.md new file mode 100644 index 00000000000..9cc9532d018 --- /dev/null +++ b/changelog/19.0/19.0.7/release_notes.md @@ -0,0 +1,7 @@ +# Release of Vitess v19.0.7 +The entire changelog for this release can be found [here](https://github.com/vitessio/vitess/blob/main/changelog/19.0/19.0.7/changelog.md). + +The release includes 24 merged Pull Requests. + +Thanks to all our contributors: @app/vitess-bot, @frouioui, @systay, @timvaillancourt, @vitess-bot + diff --git a/changelog/19.0/README.md b/changelog/19.0/README.md index cd43ee9bbd3..008c92c2aec 100644 --- a/changelog/19.0/README.md +++ b/changelog/19.0/README.md @@ -1,4 +1,8 @@ ## v19.0 +* **[19.0.7](19.0.7)** + * [Changelog](19.0.7/changelog.md) + * [Release Notes](19.0.7/release_notes.md) + * **[19.0.6](19.0.6)** * [Changelog](19.0.6/changelog.md) * [Release Notes](19.0.6/release_notes.md) diff --git a/examples/compose/docker-compose.beginners.yml b/examples/compose/docker-compose.beginners.yml index 8af25f3b173..3e5e450c1c0 100644 --- a/examples/compose/docker-compose.beginners.yml +++ b/examples/compose/docker-compose.beginners.yml @@ -58,7 +58,7 @@ services: - "3306" vtctld: - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - "15000:$WEB_PORT" - "$GRPC_PORT" @@ -81,7 +81,7 @@ services: condition: service_healthy vtgate: - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - "15099:$WEB_PORT" - "$GRPC_PORT" @@ -111,7 +111,7 @@ services: condition: service_healthy schemaload: - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 command: - sh - -c @@ -144,12 +144,12 @@ services: environment: - KEYSPACES=$KEYSPACE - GRPC_PORT=15999 - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 volumes: - .:/script vttablet100: - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - "15100:$WEB_PORT" - "$GRPC_PORT" @@ -181,7 +181,7 @@ services: retries: 15 vttablet101: - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - "15101:$WEB_PORT" - "$GRPC_PORT" @@ -213,7 +213,7 @@ services: retries: 15 vttablet102: - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - "15102:$WEB_PORT" - "$GRPC_PORT" @@ -245,7 +245,7 @@ services: retries: 15 vttablet103: - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - "15103:$WEB_PORT" - "$GRPC_PORT" @@ -277,7 +277,7 @@ services: retries: 15 vtorc: - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 command: ["sh", "-c", "/script/vtorc-up.sh"] depends_on: - vtctld @@ -307,7 +307,7 @@ services: retries: 15 vreplication: - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 volumes: - ".:/script" environment: diff --git a/examples/compose/docker-compose.yml b/examples/compose/docker-compose.yml index f4944ae6b97..d7e463046ee 100644 --- a/examples/compose/docker-compose.yml +++ b/examples/compose/docker-compose.yml @@ -75,7 +75,7 @@ services: - SCHEMA_FILES=lookup_keyspace_schema_file.sql - POST_LOAD_FILE= - EXTERNAL_DB=0 - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 volumes: - .:/script schemaload_test_keyspace: @@ -101,7 +101,7 @@ services: - SCHEMA_FILES=test_keyspace_schema_file.sql - POST_LOAD_FILE= - EXTERNAL_DB=0 - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 volumes: - .:/script set_keyspace_durability_policy: @@ -115,7 +115,7 @@ services: environment: - KEYSPACES=test_keyspace lookup_keyspace - GRPC_PORT=15999 - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 volumes: - .:/script vreplication: @@ -129,7 +129,7 @@ services: - TOPOLOGY_FLAGS=--topo_implementation consul --topo_global_server_address consul1:8500 --topo_global_root vitess/global - EXTERNAL_DB=0 - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 volumes: - .:/script vtctld: @@ -143,7 +143,7 @@ services: depends_on: external_db_host: condition: service_healthy - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - 15000:8080 - "15999" @@ -160,7 +160,7 @@ services: --normalize_queries=true ' depends_on: - vtctld - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - 15099:8080 - "15999" @@ -182,7 +182,7 @@ services: - EXTERNAL_DB=0 - DB_USER= - DB_PASS= - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - 13000:8080 volumes: @@ -217,7 +217,7 @@ services: - CMD-SHELL - curl -s --fail --show-error localhost:8080/debug/health timeout: 10s - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - 15101:8080 - "15999" @@ -254,7 +254,7 @@ services: - CMD-SHELL - curl -s --fail --show-error localhost:8080/debug/health timeout: 10s - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - 15102:8080 - "15999" @@ -291,7 +291,7 @@ services: - CMD-SHELL - curl -s --fail --show-error localhost:8080/debug/health timeout: 10s - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - 15201:8080 - "15999" @@ -328,7 +328,7 @@ services: - CMD-SHELL - curl -s --fail --show-error localhost:8080/debug/health timeout: 10s - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - 15202:8080 - "15999" @@ -365,7 +365,7 @@ services: - CMD-SHELL - curl -s --fail --show-error localhost:8080/debug/health timeout: 10s - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - 15301:8080 - "15999" @@ -402,7 +402,7 @@ services: - CMD-SHELL - curl -s --fail --show-error localhost:8080/debug/health timeout: 10s - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - 15302:8080 - "15999" diff --git a/examples/compose/vtcompose/docker-compose.test.yml b/examples/compose/vtcompose/docker-compose.test.yml index 8f88a2d08ae..72c58601a66 100644 --- a/examples/compose/vtcompose/docker-compose.test.yml +++ b/examples/compose/vtcompose/docker-compose.test.yml @@ -79,7 +79,7 @@ services: - SCHEMA_FILES=test_keyspace_schema_file.sql - POST_LOAD_FILE= - EXTERNAL_DB=0 - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 volumes: - .:/script schemaload_unsharded_keyspace: @@ -103,7 +103,7 @@ services: - SCHEMA_FILES=unsharded_keyspace_schema_file.sql - POST_LOAD_FILE= - EXTERNAL_DB=0 - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 volumes: - .:/script set_keyspace_durability_policy_test_keyspace: @@ -117,7 +117,7 @@ services: environment: - GRPC_PORT=15999 - KEYSPACES=test_keyspace - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 volumes: - .:/script set_keyspace_durability_policy_unsharded_keyspace: @@ -130,7 +130,7 @@ services: environment: - GRPC_PORT=15999 - KEYSPACES=unsharded_keyspace - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 volumes: - .:/script vreplication: @@ -144,7 +144,7 @@ services: - TOPOLOGY_FLAGS=--topo_implementation consul --topo_global_server_address consul1:8500 --topo_global_root vitess/global - EXTERNAL_DB=0 - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 volumes: - .:/script vtctld: @@ -159,7 +159,7 @@ services: depends_on: external_db_host: condition: service_healthy - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - 15000:8080 - "15999" @@ -176,7 +176,7 @@ services: ''grpc-vtgateservice'' --normalize_queries=true ' depends_on: - vtctld - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - 15099:8080 - "15999" @@ -199,7 +199,7 @@ services: - EXTERNAL_DB=0 - DB_USER= - DB_PASS= - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - 13000:8080 volumes: @@ -234,7 +234,7 @@ services: - CMD-SHELL - curl -s --fail --show-error localhost:8080/debug/health timeout: 10s - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - 15101:8080 - "15999" @@ -271,7 +271,7 @@ services: - CMD-SHELL - curl -s --fail --show-error localhost:8080/debug/health timeout: 10s - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - 15102:8080 - "15999" @@ -308,7 +308,7 @@ services: - CMD-SHELL - curl -s --fail --show-error localhost:8080/debug/health timeout: 10s - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - 15201:8080 - "15999" @@ -345,7 +345,7 @@ services: - CMD-SHELL - curl -s --fail --show-error localhost:8080/debug/health timeout: 10s - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - 15202:8080 - "15999" @@ -382,7 +382,7 @@ services: - CMD-SHELL - curl -s --fail --show-error localhost:8080/debug/health timeout: 10s - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - 15301:8080 - "15999" diff --git a/examples/compose/vtcompose/vtcompose.go b/examples/compose/vtcompose/vtcompose.go index ce04a876759..de9857f5f02 100644 --- a/examples/compose/vtcompose/vtcompose.go +++ b/examples/compose/vtcompose/vtcompose.go @@ -533,7 +533,7 @@ func generateDefaultShard(tabAlias int, shard string, keyspaceData keyspaceInfo, - op: add path: /services/init_shard_primary%[2]d value: - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 command: ["sh", "-c", "/vt/bin/vtctldclient %[5]s InitShardPrimary --force %[4]s/%[3]s %[6]s-%[2]d "] %[1]s `, dependsOn, aliases[0], shard, keyspaceData.keyspace, opts.topologyFlags, opts.cell) @@ -565,7 +565,7 @@ func generateExternalPrimary( - op: add path: /services/vttablet%[1]d value: - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - "15%[1]d:%[3]d" - "%[4]d" @@ -627,7 +627,7 @@ func generateDefaultTablet(tabAlias int, shard, role, keyspace string, dbInfo ex - op: add path: /services/vttablet%[1]d value: - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - "15%[1]d:%[4]d" - "%[5]d" @@ -665,7 +665,7 @@ func generateVtctld(opts vtOptions) string { - op: add path: /services/vtctld value: - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - "15000:%[1]d" - "%[2]d" @@ -696,7 +696,7 @@ func generateVtgate(opts vtOptions) string { - op: add path: /services/vtgate value: - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 ports: - "15099:%[1]d" - "%[2]d" @@ -738,7 +738,7 @@ func generateVTOrc(dbInfo externalDbInfo, keyspaceInfoMap map[string]keyspaceInf - op: add path: /services/vtorc value: - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 volumes: - ".:/script" environment: @@ -763,7 +763,7 @@ func generateVreplication(dbInfo externalDbInfo, opts vtOptions) string { - op: add path: /services/vreplication value: - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 volumes: - ".:/script" environment: @@ -791,7 +791,7 @@ func generateSetKeyspaceDurabilityPolicy( - op: add path: /services/set_keyspace_durability_policy_%[3]s value: - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 volumes: - ".:/script" environment: @@ -828,7 +828,7 @@ func generateSchemaload( - op: add path: /services/schemaload_%[7]s value: - image: vitess/lite:v19.0.6 + image: vitess/lite:v19.0.7 volumes: - ".:/script" environment: diff --git a/examples/operator/101_initial_cluster.yaml b/examples/operator/101_initial_cluster.yaml index 9fdb58641a3..7df159a56f9 100644 --- a/examples/operator/101_initial_cluster.yaml +++ b/examples/operator/101_initial_cluster.yaml @@ -8,14 +8,14 @@ metadata: name: example spec: images: - vtctld: vitess/lite:v19.0.6 - vtadmin: vitess/vtadmin:v19.0.6 - vtgate: vitess/lite:v19.0.6 - vttablet: vitess/lite:v19.0.6 - vtbackup: vitess/lite:v19.0.6 - vtorc: vitess/lite:v19.0.6 + vtctld: vitess/lite:v19.0.7 + vtadmin: vitess/vtadmin:v19.0.7 + vtgate: vitess/lite:v19.0.7 + vttablet: vitess/lite:v19.0.7 + vtbackup: vitess/lite:v19.0.7 + vtorc: vitess/lite:v19.0.7 mysqld: - mysql80Compatible: vitess/lite:v19.0.6 + mysql80Compatible: vitess/lite:v19.0.7 mysqldExporter: prom/mysqld-exporter:v0.11.0 cells: - name: zone1 diff --git a/examples/operator/201_customer_tablets.yaml b/examples/operator/201_customer_tablets.yaml index e61eba759bd..aa6b244b3fb 100644 --- a/examples/operator/201_customer_tablets.yaml +++ b/examples/operator/201_customer_tablets.yaml @@ -4,14 +4,14 @@ metadata: name: example spec: images: - vtctld: vitess/lite:v19.0.6 - vtadmin: vitess/vtadmin:v19.0.6 - vtgate: vitess/lite:v19.0.6 - vttablet: vitess/lite:v19.0.6 - vtbackup: vitess/lite:v19.0.6 - vtorc: vitess/lite:v19.0.6 + vtctld: vitess/lite:v19.0.7 + vtadmin: vitess/vtadmin:v19.0.7 + vtgate: vitess/lite:v19.0.7 + vttablet: vitess/lite:v19.0.7 + vtbackup: vitess/lite:v19.0.7 + vtorc: vitess/lite:v19.0.7 mysqld: - mysql80Compatible: vitess/lite:v19.0.6 + mysql80Compatible: vitess/lite:v19.0.7 mysqldExporter: prom/mysqld-exporter:v0.11.0 cells: - name: zone1 diff --git a/examples/operator/302_new_shards.yaml b/examples/operator/302_new_shards.yaml index 0690f11c9d7..d7da70d3aa4 100644 --- a/examples/operator/302_new_shards.yaml +++ b/examples/operator/302_new_shards.yaml @@ -4,14 +4,14 @@ metadata: name: example spec: images: - vtctld: vitess/lite:v19.0.6 - vtadmin: vitess/vtadmin:v19.0.6 - vtgate: vitess/lite:v19.0.6 - vttablet: vitess/lite:v19.0.6 - vtbackup: vitess/lite:v19.0.6 - vtorc: vitess/lite:v19.0.6 + vtctld: vitess/lite:v19.0.7 + vtadmin: vitess/vtadmin:v19.0.7 + vtgate: vitess/lite:v19.0.7 + vttablet: vitess/lite:v19.0.7 + vtbackup: vitess/lite:v19.0.7 + vtorc: vitess/lite:v19.0.7 mysqld: - mysql80Compatible: vitess/lite:v19.0.6 + mysql80Compatible: vitess/lite:v19.0.7 mysqldExporter: prom/mysqld-exporter:v0.11.0 cells: - name: zone1 diff --git a/examples/operator/306_down_shard_0.yaml b/examples/operator/306_down_shard_0.yaml index 4491776cdc2..23465bfccb4 100644 --- a/examples/operator/306_down_shard_0.yaml +++ b/examples/operator/306_down_shard_0.yaml @@ -4,14 +4,14 @@ metadata: name: example spec: images: - vtctld: vitess/lite:v19.0.6 - vtadmin: vitess/vtadmin:v19.0.6 - vtgate: vitess/lite:v19.0.6 - vttablet: vitess/lite:v19.0.6 - vtbackup: vitess/lite:v19.0.6 - vtorc: vitess/lite:v19.0.6 + vtctld: vitess/lite:v19.0.7 + vtadmin: vitess/vtadmin:v19.0.7 + vtgate: vitess/lite:v19.0.7 + vttablet: vitess/lite:v19.0.7 + vtbackup: vitess/lite:v19.0.7 + vtorc: vitess/lite:v19.0.7 mysqld: - mysql80Compatible: vitess/lite:v19.0.6 + mysql80Compatible: vitess/lite:v19.0.7 mysqldExporter: prom/mysqld-exporter:v0.11.0 cells: - name: zone1 diff --git a/go/mysql/flavor_mysql.go b/go/mysql/flavor_mysql.go index f83f087582c..dd108a74225 100644 --- a/go/mysql/flavor_mysql.go +++ b/go/mysql/flavor_mysql.go @@ -337,15 +337,20 @@ WHERE t.table_schema = database() GROUP BY t.table_name, t.table_type, t.create_time, t.table_comment` // TablesWithSize80 is a query to select table along with size for mysql 8.0 -// // Note the following: -// - We use a single query to fetch both partitioned and non-partitioned tables. This is because -// accessing `information_schema.innodb_tablespaces` is expensive on servers with many tablespaces, -// and every query that loads the table needs to perform full table scans on it. Doing a single -// table scan is more efficient than doing more than one. -// - We utilize `INFORMATION_SCHEMA`.`TABLES`.`CREATE_OPTIONS` column to do early pruning before the JOIN. // - `TABLES`.`TABLE_NAME` has `utf8mb4_0900_ai_ci` collation. `INNODB_TABLESPACES`.`NAME` has `utf8mb3_general_ci`. // We normalize the collation to get better query performance (we force the casting at the time of our choosing) +// - InnoDB has different table names than MySQL does, in particular for partitioned tables. As far as InnoDB +// is concerned, each partition is its own table. +// - We use a `UNION ALL` approach to handle two distinct scenarios: tables that are partitioned and those that are not. +// Since we `LEFT JOIN` from `TABLES` to `INNODB_TABLESPACES`, we know we already do full table scan on `TABLES`. We therefore +// don't mind spending some extra computation time (as in `CONCAT(t.table_schema, '/', t.table_name, '#p#%') COLLATE utf8mb3_general_ci`) +// to make things easier for the JOIN. +// - We utilize `INFORMATION_SCHEMA`.`TABLES`.`CREATE_OPTIONS` column to tell if the table is partitioned or not. The column +// may be `NULL` or may have multiple attributes, one of which is "partitioned", which we are looking for. +// - In a partitioned table, InnoDB will return multiple rows for the same table name, one for each partition, which we successively SUM. +// We also `SUM` the sizes in the non-partitioned case. This is not because we need to, but because it makes the query +// symmetric and less prone to future edit errors. const TablesWithSize80 = `SELECT t.table_name, t.table_type, UNIX_TIMESTAMP(t.create_time), @@ -353,10 +358,24 @@ const TablesWithSize80 = `SELECT t.table_name, SUM(i.file_size), SUM(i.allocated_size) FROM information_schema.tables t - LEFT JOIN information_schema.innodb_tablespaces i - ON i.name LIKE CONCAT(t.table_schema, '/', t.table_name, IF(t.create_options <=> 'partitioned', '#p#%', '')) COLLATE utf8mb3_general_ci + LEFT JOIN (SELECT name, file_size, allocated_size FROM information_schema.innodb_tablespaces WHERE name LIKE CONCAT(database(), '/%')) i + ON i.name = CONCAT(t.table_schema, '/', t.table_name) COLLATE utf8mb3_general_ci WHERE - t.table_schema = database() + t.table_schema = database() AND IFNULL(t.create_options, '') NOT LIKE '%partitioned%' + GROUP BY + t.table_schema, t.table_name, t.table_type, t.create_time, t.table_comment +UNION ALL +SELECT t.table_name, + t.table_type, + UNIX_TIMESTAMP(t.create_time), + t.table_comment, + SUM(i.file_size), + SUM(i.allocated_size) + FROM information_schema.tables t + LEFT JOIN (SELECT name, file_size, allocated_size FROM information_schema.innodb_tablespaces WHERE name LIKE CONCAT(database(), '/%')) i + ON i.name LIKE (CONCAT(t.table_schema, '/', t.table_name, '#p#%') COLLATE utf8mb3_general_ci) + WHERE + t.table_schema = database() AND t.create_options LIKE '%partitioned%' GROUP BY t.table_schema, t.table_name, t.table_type, t.create_time, t.table_comment ` diff --git a/go/vt/mysqlctl/backup_blackbox_race_test.go b/go/vt/mysqlctl/backup_blackbox_race_test.go new file mode 100644 index 00000000000..1cbae4bd398 --- /dev/null +++ b/go/vt/mysqlctl/backup_blackbox_race_test.go @@ -0,0 +1,152 @@ +//go:build !race + +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package mysqlctl_test is the blackbox tests for package mysqlctl. +package mysqlctl_test + +import ( + "fmt" + "os" + "path" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/mysql/fakesqldb" + "vitess.io/vitess/go/test/utils" + "vitess.io/vitess/go/vt/logutil" + "vitess.io/vitess/go/vt/mysqlctl" + "vitess.io/vitess/go/vt/mysqlctl/backupstats" + "vitess.io/vitess/go/vt/mysqlctl/filebackupstorage" + "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/proto/vttime" + "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/topo/memorytopo" +) + +// This test triggers a certain code path that only happens when a backup file fails to be backed up, +// only and only if, all the other backup files have either started or finished. When we reach +// this scenario, files no longer try to acquire the semaphore and thus the backup cannot fail +// because of context deadline when acquiring it. At this point, the only place where the backup +// can fail, is if the return of be.backupFiles fails, and we record the error correctly. +// This test specifically test this scenario and arose because of issue https://github.com/vitessio/vitess/issues/17063 +// The test does: +// 1. Create the backup and data directory +// 2. Create a keyspace and shard +// 3. Already create the last backup file that would be created +// 4. Remove all permissions on this file +// 5. Execute the restore +// 6. The restore must fail due to an error on file number 3 ("cannot add file: 3") +// +// This test is extracted into its own file that won't be run if we do 'go test -race' as this test +// exposes an old race condition that will be fixed after https://github.com/vitessio/vitess/pull/17062 +// Link to the race condition issue: https://github.com/vitessio/vitess/issues/17065 +func TestExecuteBackupWithFailureOnLastFile(t *testing.T) { + ctx := utils.LeakCheckContext(t) + + // Set up local backup directory + id := fmt.Sprintf("%d", time.Now().UnixNano()) + backupRoot := fmt.Sprintf("testdata/builtinbackup_test_%s", id) + filebackupstorage.FileBackupStorageRoot = backupRoot + require.NoError(t, createBackupDir(backupRoot, "innodb", "log", "datadir")) + dataDir := path.Join(backupRoot, "datadir") + // Add some files under data directory to force backup to execute semaphore acquire inside + // backupFiles() method (https://github.com/vitessio/vitess/blob/main/go/vt/mysqlctl/builtinbackupengine.go#L483). + require.NoError(t, createBackupDir(dataDir, "test1")) + require.NoError(t, createBackupDir(dataDir, "test2")) + require.NoError(t, createBackupFiles(path.Join(dataDir, "test1"), 2, "ibd")) + require.NoError(t, createBackupFiles(path.Join(dataDir, "test2"), 2, "ibd")) + defer os.RemoveAll(backupRoot) + + needIt, err := needInnoDBRedoLogSubdir() + require.NoError(t, err) + if needIt { + fpath := path.Join("log", mysql.DynamicRedoLogSubdir) + if err := createBackupDir(backupRoot, fpath); err != nil { + require.Failf(t, err.Error(), "failed to create directory: %s", fpath) + } + } + + // Set up topo + keyspace, shard := "mykeyspace", "-" + ts := memorytopo.NewServer(ctx, "cell1") + defer ts.Close() + + require.NoError(t, ts.CreateKeyspace(ctx, keyspace, &topodata.Keyspace{})) + require.NoError(t, ts.CreateShard(ctx, keyspace, shard)) + + tablet := topo.NewTablet(100, "cell1", "mykeyspace-00-80-0100") + tablet.Keyspace = keyspace + tablet.Shard = shard + + require.NoError(t, ts.CreateTablet(ctx, tablet)) + + _, err = ts.UpdateShardFields(ctx, keyspace, shard, func(si *topo.ShardInfo) error { + si.PrimaryAlias = &topodata.TabletAlias{Uid: 100, Cell: "cell1"} + + now := time.Now() + si.PrimaryTermStartTime = &vttime.Time{Seconds: int64(now.Second()), Nanoseconds: int32(now.Nanosecond())} + + return nil + }) + + require.NoError(t, err) + + be := &mysqlctl.BuiltinBackupEngine{} + bh := filebackupstorage.NewBackupHandle(nil, "", "", false) + // Spin up a fake daemon to be used in backups. It needs to be allowed to receive: + // "STOP REPLICA", "START REPLICA", in that order. + fakedb := fakesqldb.New(t) + defer fakedb.Close() + mysqld := mysqlctl.NewFakeMysqlDaemon(fakedb) + defer mysqld.Close() + mysqld.ExpectedExecuteSuperQueryList = []string{"STOP SLAVE", "START SLAVE"} + + // With this setup, 4 backup files will be created (0, 1, 2, 3). For the last file (3), we create + // it in advance and remove all permission on the file so that the backup be.ExecuteBackup will not + // be able to override the file and thus will fail. Triggering the error mechanism after calling be.backupFile. + lastBackupFile := path.Join(backupRoot, "3") + f, err := os.Create(lastBackupFile) + require.NoError(t, err) + _, err = f.Write(make([]byte, 1024)) + require.NoError(t, err) + require.NoError(t, f.Chmod(0444)) + require.NoError(t, f.Close()) + + backupResult, err := be.ExecuteBackup(ctx, mysqlctl.BackupParams{ + Logger: logutil.NewConsoleLogger(), + Mysqld: mysqld, + Cnf: &mysqlctl.Mycnf{ + InnodbDataHomeDir: path.Join(backupRoot, "innodb"), + InnodbLogGroupHomeDir: path.Join(backupRoot, "log"), + DataDir: path.Join(backupRoot, "datadir"), + }, + Stats: backupstats.NewFakeStats(), + Concurrency: 4, + HookExtraEnv: map[string]string{}, + TopoServer: ts, + Keyspace: keyspace, + Shard: shard, + MysqlShutdownTimeout: mysqlShutdownTimeout, + }, bh) + + require.ErrorContains(t, err, "cannot add file: 3") + require.Equal(t, mysqlctl.BackupUnusable, backupResult) +} diff --git a/go/vt/mysqlctl/builtinbackupengine.go b/go/vt/mysqlctl/builtinbackupengine.go index 4b9a0fa4269..af6b8fdbd85 100644 --- a/go/vt/mysqlctl/builtinbackupengine.go +++ b/go/vt/mysqlctl/builtinbackupengine.go @@ -652,9 +652,8 @@ func (be *BuiltinBackupEngine) backupFiles( // Backup the individual file. name := fmt.Sprintf("%v", i) - err := be.backupFile(ctxCancel, params, bh, fe, name) - if err != nil { - bh.RecordError(acqErr) + if err := be.backupFile(ctxCancel, params, bh, fe, name); err != nil { + bh.RecordError(err) cancel() } }(i) diff --git a/go/vt/servenv/version.go b/go/vt/servenv/version.go index 79866937497..ec7f0c9f7fe 100644 --- a/go/vt/servenv/version.go +++ b/go/vt/servenv/version.go @@ -19,4 +19,4 @@ package servenv // DO NOT EDIT // THIS FILE IS AUTO-GENERATED DURING NEW RELEASES BY THE VITESS-RELEASER -const versionName = "19.0.7-SNAPSHOT" +const versionName = "19.0.8-SNAPSHOT" diff --git a/go/vt/vttablet/onlineddl/executor.go b/go/vt/vttablet/onlineddl/executor.go index 910f52a0333..5dd1811affd 100644 --- a/go/vt/vttablet/onlineddl/executor.go +++ b/go/vt/vttablet/onlineddl/executor.go @@ -895,8 +895,8 @@ func (e *Executor) cutOverVReplMigration(ctx context.Context, s *VReplStream, sh migrationCutOverThreshold := getMigrationCutOverThreshold(onlineDDL) - waitForPos := func(s *VReplStream, pos replication.Position) error { - ctx, cancel := context.WithTimeout(ctx, migrationCutOverThreshold) + waitForPos := func(s *VReplStream, pos replication.Position, timeout time.Duration) error { + ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() // Wait for target to reach the up-to-date pos if err := tmClient.VReplicationWaitForPos(ctx, tablet.Tablet, s.id, replication.EncodePosition(pos)); err != nil { @@ -954,8 +954,12 @@ func (e *Executor) cutOverVReplMigration(ctx context.Context, s *VReplStream, sh return err } e.updateMigrationStage(ctx, onlineDDL.UUID, "waiting for post-sentry pos: %v", replication.EncodePosition(postSentryPos)) - if err := waitForPos(s, postSentryPos); err != nil { - return err + // We have not yet locked anything, stopped anything, or done anything that otherwise + // impacts query serving so we wait for a multiple of the cutover threshold here, with + // that variable primarily serving to limit the max time we later spend waiting for + // a position again AFTER we've taken the locks and table access is blocked. + if err := waitForPos(s, postSentryPos, migrationCutOverThreshold*3); err != nil { + return vterrors.Wrapf(err, "failed waiting for pos after sentry creation") } e.updateMigrationStage(ctx, onlineDDL.UUID, "post-sentry pos reached") } @@ -1129,7 +1133,7 @@ func (e *Executor) cutOverVReplMigration(ctx context.Context, s *VReplStream, sh } e.updateMigrationStage(ctx, onlineDDL.UUID, "waiting for post-lock pos: %v", replication.EncodePosition(postWritesPos)) - if err := waitForPos(s, postWritesPos); err != nil { + if err := waitForPos(s, postWritesPos, migrationCutOverThreshold); err != nil { e.updateMigrationStage(ctx, onlineDDL.UUID, "timeout while waiting for post-lock pos: %v", err) return err } diff --git a/java/client/pom.xml b/java/client/pom.xml index f3708d00e5d..ece003dca3c 100644 --- a/java/client/pom.xml +++ b/java/client/pom.xml @@ -5,7 +5,7 @@ io.vitess vitess-parent - 19.0.7-SNAPSHOT + 19.0.8-SNAPSHOT vitess-client diff --git a/java/example/pom.xml b/java/example/pom.xml index e9597ce2d83..a151f8e99b8 100644 --- a/java/example/pom.xml +++ b/java/example/pom.xml @@ -5,7 +5,7 @@ io.vitess vitess-parent - 19.0.7-SNAPSHOT + 19.0.8-SNAPSHOT vitess-example diff --git a/java/grpc-client/pom.xml b/java/grpc-client/pom.xml index f0061b6f4af..f0f7ed09abe 100644 --- a/java/grpc-client/pom.xml +++ b/java/grpc-client/pom.xml @@ -5,7 +5,7 @@ io.vitess vitess-parent - 19.0.7-SNAPSHOT + 19.0.8-SNAPSHOT vitess-grpc-client diff --git a/java/jdbc/pom.xml b/java/jdbc/pom.xml index 57ab1b9cf18..f58a696767e 100644 --- a/java/jdbc/pom.xml +++ b/java/jdbc/pom.xml @@ -5,7 +5,7 @@ io.vitess vitess-parent - 19.0.7-SNAPSHOT + 19.0.8-SNAPSHOT vitess-jdbc diff --git a/java/pom.xml b/java/pom.xml index ee5d849fa3a..3079f082cd5 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -11,7 +11,7 @@ io.vitess vitess-parent - 19.0.7-SNAPSHOT + 19.0.8-SNAPSHOT pom Vitess Java Client libraries [Parent]