From 47a9d7685edf28d397d29dd4821f1ca4d2e8a34e Mon Sep 17 00:00:00 2001 From: anonymouzz Date: Tue, 22 Jul 2014 22:09:03 +0700 Subject: [PATCH 1/3] Add argument --single-transaction (-s): make all migration in single transaction --- bin/py-mysql2pgsql | 6 ++++++ mysql2pgsql/lib/postgres_db_writer.py | 14 ++++++++++---- mysql2pgsql/mysql2pgsql.py | 5 +++-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/bin/py-mysql2pgsql b/bin/py-mysql2pgsql index 6afcc5a..6fe07db 100755 --- a/bin/py-mysql2pgsql +++ b/bin/py-mysql2pgsql @@ -22,6 +22,12 @@ if __name__ == '__main__': default='mysql2pgsql.yml', help='Location of configuration file (default: %(default)s). If none exists at that path, one will be created for you.', ) + parser.add_argument( + '-s', '--single-transaction', + default=False, + action='store_true', + help='Make all changes in single transaction (default: False)', + ) parser.add_argument( '-V', '--version', action='store_true', diff --git a/mysql2pgsql/lib/postgres_db_writer.py b/mysql2pgsql/lib/postgres_db_writer.py index 2a67c15..43732db 100644 --- a/mysql2pgsql/lib/postgres_db_writer.py +++ b/mysql2pgsql/lib/postgres_db_writer.py @@ -28,11 +28,12 @@ class FileObjFaker(object): - `processor`: - `verbose`: whether or not to log progress to :py:obj:`stdout` """ - def __init__(self, table, data, processor, verbose=False): + def __init__(self, table, data, processor, verbose=False, single_transaction=False): self.data = iter(data) self.table = table self.processor = processor self.verbose = verbose + self.single_transaction = single_transaction if verbose: self.idx = 1 @@ -68,9 +69,10 @@ def readline(self, *args, **kwargs): def read(self, *args, **kwargs): return self.readline(*args, **kwargs) - def __init__(self, db_options, verbose=False, *args, **kwargs): + def __init__(self, db_options, verbose=False, single_transaction=False, *args, **kwargs): super(PostgresDbWriter, self).__init__(*args, **kwargs) self.verbose = verbose + self.single_transaction = single_transaction self.db_options = { 'host': str(db_options['hostname']), 'port': db_options.get('port', 5432), @@ -107,7 +109,8 @@ def execute(self, sql, args=(), many=False): cur.executemany(sql, args) else: cur.execute(sql, args) - self.conn.commit() + if not self.single_transaction: + self.conn.commit() def copy_from(self, file_obj, table_name, columns): with closing(self.conn.cursor()) as cur: @@ -116,10 +119,13 @@ def copy_from(self, file_obj, table_name, columns): columns=columns ) - self.conn.commit() + if not self.single_transaction: + self.conn.commit() def close(self): """Closes connection to the PostgreSQL server""" + if self.single_transaction: + self.conn.commit() self.conn.close() def exists(self, relname): diff --git a/mysql2pgsql/mysql2pgsql.py b/mysql2pgsql/mysql2pgsql.py index c2136f0..e2cccd0 100644 --- a/mysql2pgsql/mysql2pgsql.py +++ b/mysql2pgsql/mysql2pgsql.py @@ -25,12 +25,13 @@ def convert(self): if self.file_options['destination']['file']: writer = PostgresFileWriter(self._get_file(self.file_options['destination']['file']), - self.run_options.verbose, + self.run_options.verbose, tz=self.file_options.get('timezone', False), index_prefix=self.file_options.get("index_prefix", '')) else: writer = PostgresDbWriter(self.file_options['destination']['postgres'], - self.run_options.verbose, + self.run_options.verbose, + self.run_options.single_transaction, tz=self.file_options.get('timezone', False), index_prefix=self.file_options.get("index_prefix", '')) From 665a6dc9cfcffea9feabbff31fccb2349411cbfc Mon Sep 17 00:00:00 2001 From: anonymouzz Date: Thu, 28 Aug 2014 17:13:14 +0700 Subject: [PATCH 2/3] Update README.rst --- README.rst | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 16cfbd7..9b33857 100644 --- a/README.rst +++ b/README.rst @@ -80,11 +80,13 @@ Looking for help? Tool for migrating/converting data from mysql to postgresql. optional arguments: - -h, --help show this help message and exit - -v, --verbose Show progress of data migration. - -f FILE, --file FILE Location of configuration file (default: - mysql2pgsql.yml). If none exists at that path, - one will be created for you. + -h, --help Show this help message and exit + -v, --verbose Show progress of data migration. + -f FILE, --file FILE Location of configuration file (default: + mysql2pgsql.yml). If none exists at that path, + one will be created for you. + -s, --signle-transaction Make data migration in signle transaction + (default: False) Don't worry if this is your first time, it'll be gentle. @@ -172,6 +174,11 @@ sequences out of whack. When this happens, you may get IntegrityErrors about your primary keys saying things like, "duplicate key value violates unique constraint." See `this page `_ for a fix +In some cases, you may get: IngegrityError: insert or update on table +'refs_table' violates freign key constraint 'id_refs_main_table_id'. +DETAIL: Key (id)=123 is not present in table 'main_table'. +In this case, `--single-transaction` key may help. + Due to different naming conventions in mysql an postgrsql, there is a chance that the tool generates index names that collide with table names. This can be circumvented by setting index_prefix. From 3fbbcb9fcd2a5dd5b3f01bdbd39acf0694ac0168 Mon Sep 17 00:00:00 2001 From: anonymouzz Date: Thu, 28 Aug 2014 17:13:58 +0700 Subject: [PATCH 3/3] Update py-mysql2pgsql --- bin/py-mysql2pgsql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/py-mysql2pgsql b/bin/py-mysql2pgsql index 6fe07db..d6374b6 100755 --- a/bin/py-mysql2pgsql +++ b/bin/py-mysql2pgsql @@ -26,7 +26,7 @@ if __name__ == '__main__': '-s', '--single-transaction', default=False, action='store_true', - help='Make all changes in single transaction (default: False)', + help='Make data migration in signle transaction (default: False)', ) parser.add_argument( '-V', '--version',