Merge pull request #1962 from Automattic/esp-72/mydumper-support

Mydumper integration

__fixtures__/dev-env-e2e/mydumper-detection.expected.sql

@@ -0,0 +1,16 @@
+
+-- metadata.header -1
+# Started dump at: 2024-07-26 03:00:36
+[config]
+quote_character = BACKTICK
+
+[myloader_session_variables]
+SQL_MODE='NO_AUTO_VALUE_ON_ZERO,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION' /*!40101
+
+
+-- some_db-schema-create.sql -1
+/*!40101 SET NAMES utf8mb4*/;
+/*!40014 SET FOREIGN_KEY_CHECKS=0*/;
+/*!40101 SET SQL_MODE='NO_AUTO_VALUE_ON_ZERO,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'*/;
+/*!40103 SET TIME_ZONE='+00:00' */;
+CREATE DATABASE /*!32312 IF NOT EXISTS*/ `some_db` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci */ /*!80016 DEFAULT ENCRYPTION='N' */;

__fixtures__/dev-env-e2e/mydumper-detection.sql

@@ -0,0 +1,16 @@
+
+-- metadata.header 198
+# Started dump at: 2024-07-26 03:00:36
+[config]
+quote_character = BACKTICK
+
+[myloader_session_variables]
+SQL_MODE='NO_AUTO_VALUE_ON_ZERO,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION' /*!40101
+
+
+-- some_db-schema-create.sql 370
+/*!40101 SET NAMES utf8mb4*/;
+/*!40014 SET FOREIGN_KEY_CHECKS=0*/;
+/*!40101 SET SQL_MODE='NO_AUTO_VALUE_ON_ZERO,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'*/;
+/*!40103 SET TIME_ZONE='+00:00' */;
+CREATE DATABASE /*!32312 IF NOT EXISTS*/ `some_db` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci */ /*!80016 DEFAULT ENCRYPTION='N' */;

__fixtures__/dev-env-e2e/mysqldump-detection.sql

@@ -0,0 +1,16 @@
+-- MySQL dump 10.13  Distrib 8.0.28, for Linux (x86_64)
+--
+-- Host: localhost    Database: some_db
+-- ------------------------------------------------------
+-- Server version	8.0.28
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!50503 SET NAMES utf8mb4 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;

__tests__/commands/dev-env-sync-sql.ts

@@ -1,74 +1,24 @@
 import { replace } from '@automattic/vip-search-replace';
-import fs, { ReadStream } from 'fs';
+import fs from 'fs';
 import Lando from 'lando';
-import { WriteStream } from 'node:fs';
-import { Interface } from 'node:readline';
-import { PassThrough } from 'stream';
+import path from 'path';
 
 import { DevEnvImportSQLCommand } from '../../src/commands/dev-env-import-sql';
 import { DevEnvSyncSQLCommand } from '../../src/commands/dev-env-sync-sql';
 import { ExportSQLCommand } from '../../src/commands/export-sql';
-import { unzipFile } from '../../src/lib/client-file-uploader';
-import { getReadInterface } from '../../src/lib/validations/line-by-line';
-
-/**
- *
- * @param {Array<{name, data}>} eventArgs Event arguments
- * @param {timeout}             timeout
- *
- * @return {Stream} A passthrough stream
- */
-function getMockStream( eventArgs: { name: string; data?: string }[], timeout = 10 ) {
-	const mockStream = new PassThrough();
-
-	if ( ! eventArgs ) {
-		eventArgs = [ { name: 'finish' } ];
-	}
-
-	// Leave 10ms of room for the listeners to setup
-	setTimeout( () => {
-		eventArgs.forEach( ( { name, data } ) => {
-			mockStream.emit( name, data );
-		} );
-	}, timeout );
-
-	return mockStream;
-}
-
-const mockReadStream: ReadStream = getMockStream(
-	[ { name: 'finish' }, { name: 'data', data: 'data' } ],
-	10
-) as unknown as ReadStream;
-const mockWriteStream: WriteStream = getMockStream(
-	[ { name: 'finish' } ],
-	20
-) as unknown as WriteStream;
-
-jest.spyOn( fs, 'createReadStream' ).mockReturnValue( mockReadStream );
-jest.spyOn( fs, 'createWriteStream' ).mockReturnValue( mockWriteStream );
-jest.spyOn( fs, 'renameSync' ).mockImplementation( () => {} );
-jest.mock( '@automattic/vip-search-replace', () => {
-	return {
-		replace: jest.fn(),
-	};
-} );
-jest.mock( '../../src/lib/client-file-uploader', () => {
-	return {
-		unzipFile: jest.fn(),
-	};
-} );
+import * as clientFileUploader from '../../src/lib/client-file-uploader';
 
-jest.mock( '../../src/lib/validations/line-by-line', () => {
+jest.mock( '@automattic/vip-search-replace', () => {
+	// eslint-disable-next-line @typescript-eslint/no-var-requires
+	const { PassThrough } = require( 'node:stream' ) as typeof import('node:stream');
 	return {
-		getReadInterface: jest.fn(),
+		replace: jest.fn( ( ...args ) => {
+			return Promise.resolve( new PassThrough().pipe( args[ 0 ] ) );
+		} ),
 	};
 } );
 
-jest.mocked( replace ).mockResolvedValue( mockReadStream );
-jest.mocked( unzipFile ).mockResolvedValue();
-jest
-	.mocked( getReadInterface )
-	.mockResolvedValue( getMockStream( [ { name: 'close' } ], 100 ) as unknown as Interface );
+jest.spyOn( clientFileUploader, 'unzipFile' );
 
 jest.spyOn( console, 'log' ).mockImplementation( () => {} );
 
@@ -126,17 +76,55 @@ describe( 'commands/DevEnvSyncSQLCommand', () => {
 	} );
 
 	describe( '.runSearchReplace', () => {
-		it( 'should run search-replace operation on the SQL file', async () => {
+		it( 'should run search-replace operation on the mysqldump file', async () => {
 			const cmd = new DevEnvSyncSQLCommand( app, env, 'test-slug', lando );
+			fs.copyFileSync(
+				path.join( __dirname, '../../__fixtures__/dev-env-e2e/mysqldump-detection.sql.gz' ),
+				cmd.gzFile
+			);
+			fs.copyFileSync(
+				path.join( __dirname, '../../__fixtures__/dev-env-e2e/mysqldump-detection.sql' ),
+				cmd.sqlFile
+			);
+			await cmd.initSqlDumpType();
 			cmd.searchReplaceMap = { 'test.go-vip.com': 'test-slug.vipdev.lndo.site' };
 			cmd.slug = 'test-slug';
 
 			await cmd.runSearchReplace();
-			expect( replace ).toHaveBeenCalledWith( mockReadStream, [
+			expect( replace ).toHaveBeenCalledWith( expect.any( Object ), [
 				'test.go-vip.com',
 				'test-slug.vipdev.lndo.site',
 			] );
 		} );
+
+		it( 'should run search-replace operation on the mydumper file', async () => {
+			const cmd = new DevEnvSyncSQLCommand( app, env, 'test-slug', lando );
+			fs.copyFileSync(
+				path.join( __dirname, '../../__fixtures__/dev-env-e2e/mydumper-detection.sql.gz' ),
+				cmd.gzFile
+			);
+			fs.copyFileSync(
+				path.join( __dirname, '../../__fixtures__/dev-env-e2e/mydumper-detection.sql' ),
+				cmd.sqlFile
+			);
+			await cmd.initSqlDumpType();
+			cmd.searchReplaceMap = { 'test.go-vip.com': 'test-slug.vipdev.lndo.site' };
+			cmd.slug = 'test-slug';
+
+			await cmd.runSearchReplace();
+			expect( replace ).toHaveBeenCalledWith( expect.any( Object ), [
+				'test.go-vip.com',
+				'test-slug.vipdev.lndo.site',
+			] );
+
+			const fileContentExpected = fs.readFileSync(
+				path.join( __dirname, '../../__fixtures__/dev-env-e2e/mydumper-detection.expected.sql' ),
+				'utf8'
+			);
+			const fileContent = fs.readFileSync( cmd.sqlFile, 'utf8' );
+
+			expect( fileContent ).toBe( fileContentExpected );
+		} );
 	} );
 
 	describe( '.runImport', () => {
@@ -159,6 +147,14 @@ describe( 'commands/DevEnvSyncSQLCommand', () => {
 		const importSpy = jest.spyOn( syncCommand, 'runImport' );
 
 		beforeAll( () => {
+			fs.copyFileSync(
+				path.join( __dirname, '../../__fixtures__/dev-env-e2e/mysqldump-detection.sql.gz' ),
+				syncCommand.gzFile
+			);
+			fs.copyFileSync(
+				path.join( __dirname, '../../__fixtures__/dev-env-e2e/mysqldump-detection.sql' ),
+				syncCommand.sqlFile
+			);
 			exportSpy.mockResolvedValue();
 			searchReplaceSpy.mockResolvedValue();
 			importSpy.mockResolvedValue();
@@ -174,7 +170,7 @@ describe( 'commands/DevEnvSyncSQLCommand', () => {
 			await syncCommand.run();
 
 			expect( exportSpy ).toHaveBeenCalled();
-			expect( unzipFile ).toHaveBeenCalled();
+			expect( clientFileUploader.unzipFile ).toHaveBeenCalled();
 			expect( generateSearchReplaceMapSpy ).toHaveBeenCalled();
 			expect( searchReplaceSpy ).toHaveBeenCalled();
 			expect( importSpy ).toHaveBeenCalled();

assets/dev-env.lando.template.yml.ejs

@@ -271,6 +271,13 @@ tooling:
     cmd:
       - wp
 
+  db-myloader:
+    service: php
+    description: "Run mydumper's myloader to import database dumps generated by mydumper"
+    user: root
+    cmd:
+      - myloader -h database -u wordpress -p wordpress --database wordpress
+
   db:
     service: php
     description: "Connect to the DB using mysql client (e.g. allow to run imports)"

src/bin/vip-dev-env-import-sql.js

@@ -2,6 +2,7 @@
 
 import { DevEnvImportSQLCommand } from '../commands/dev-env-import-sql';
 import command from '../lib/cli/command';
+import { getSqlDumpDetails } from '../lib/database';
 import {
 	getEnvTrackingInfo,
 	handleCLIException,
@@ -66,9 +67,16 @@ command( {
 	.argv( process.argv, async ( unmatchedArgs, opt ) => {
 		const [ fileName ] = unmatchedArgs;
 		const slug = await getEnvironmentName( opt );
+		if ( opt.searchReplace && ! Array.isArray( opt.searchReplace ) ) {
+			opt.searchReplace = [ opt.searchReplace ];
+		}
 		const cmd = new DevEnvImportSQLCommand( fileName, opt, slug );
+		const dumpDetails = await getSqlDumpDetails( fileName );
 		const trackingInfo = getEnvTrackingInfo( cmd.slug );
-		const trackerFn = makeCommandTracker( 'dev_env_import_sql', trackingInfo );
+		const trackerFn = makeCommandTracker( 'dev_env_import_sql', {
+			...trackingInfo,
+			sqldump_type: dumpDetails.type,
+		} );
 		await trackerFn( 'execute' );
 
 		try {

src/commands/dev-env-import-sql.ts

@@ -1,8 +1,10 @@
 import chalk from 'chalk';
 import fs from 'fs';
+import os from 'os';
 
 import * as exit from '../lib/cli/exit';
 import { getFileMeta, unzipFile } from '../lib/client-file-uploader';
+import { getSqlDumpDetails, SqlDumpDetails, SqlDumpType } from '../lib/database';
 import {
 	processBooleanOption,
 	validateDependencies,
@@ -43,6 +45,9 @@ export class DevEnvImportSQLCommand {
 
 		validateImportFileExtension( this.fileName );
 
+		const dumpDetails = await getSqlDumpDetails( this.fileName );
+		const isMyDumper = dumpDetails.type === SqlDumpType.MYDUMPER;
+
 		// Check if file is compressed and if so, extract the
 		const fileMeta = await getFileMeta( this.fileName );
 		if ( fileMeta.isCompressed ) {
@@ -83,13 +88,13 @@ export class DevEnvImportSQLCommand {
 			const expectedDomain = `${ this.slug }.${ lando.config.domain }`;
 			await validateSQL( resolvedPath, {
 				isImport: false,
-				skipChecks: [],
+				skipChecks: isMyDumper ? [ 'dropTable', 'dropDB' ] : [],
 				extraCheckParams: { siteHomeUrlLando: expectedDomain },
 			} );
 		}
 
 		const fd = await fs.promises.open( resolvedPath, 'r' );
-		const importArg = this.getImportArgs();
+		const importArg = this.getImportArgs( dumpDetails );
 
 		const origIsTTY = process.stdin.isTTY;
 
@@ -131,10 +136,27 @@ export class DevEnvImportSQLCommand {
 		await addAdminUser( lando, this.slug );
 	}
 
-	public getImportArgs() {
-		const importArg = [ 'db', '--disable-auto-rehash' ].concat(
+	public getImportArgs( dumpDetails: SqlDumpDetails ) {
+		let importArg = [ 'db', '--disable-auto-rehash' ].concat(
 			this.options.quiet ? '--silent' : []
 		);
+		const threadCount = Math.max( os.cpus().length - 2, 1 );
+		if ( dumpDetails.type === SqlDumpType.MYDUMPER ) {
+			importArg = [
+				'db-myloader',
+				'--overwrite-tables',
+				`--source-db=${ dumpDetails.sourceDb }`,
+				`--threads=${ threadCount }`,
+				'--max-threads-for-schema-creation=10',
+				'--max-threads-for-index-creation=10',
+				'--skip-triggers',
+				'--skip-post',
+				'--innodb-optimize-keys',
+				'--checksum=SKIP',
+				'--metadata-refresh-interval=2000000',
+				'--stream',
+			].concat( this.options.quiet ? [ '--verbose=0' ] : [ '--verbose=3' ] );
+		}
 
 		return importArg;
 	}