Add argument --single-transaction (-s): make all migration in single tra...

README.rst

@@ -80,11 +80,13 @@ Looking for help?
     Tool for migrating/converting data from mysql to postgresql.
 
     optional arguments:
-      -h, --help            show this help message and exit
-      -v, --verbose         Show progress of data migration.
-      -f FILE, --file FILE  Location of configuration file (default:
-                            mysql2pgsql.yml). If none exists at that path,
-                            one will be created for you.
+      -h, --help                 Show this help message and exit
+      -v, --verbose              Show progress of data migration.
+      -f FILE, --file FILE       Location of configuration file (default:
+                                   mysql2pgsql.yml). If none exists at that path,
+                                   one will be created for you.
+      -s, --signle-transaction   Make data migration in signle transaction
+                                   (default: False)
 
 
 Don't worry if this is your first time, it'll be gentle.
@@ -172,6 +174,11 @@ sequences out of whack. When this happens, you may get IntegrityErrors
 about your primary keys saying things like, "duplicate key value violates 
 unique constraint." See `this page <https://wiki.postgresql.org/wiki/Fixing_Sequences>`_ for a fix
 
+In some cases, you may get:   IngegrityError: insert or update on table 
+'refs_table' violates freign key constraint 'id_refs_main_table_id'. 
+DETAIL: Key (id)=123 is not present in table 'main_table'.
+In this case, `--single-transaction` key may help.
+
 Due to different naming conventions in mysql an postgrsql, there is a chance
 that the tool generates index names that collide with table names. This can
 be circumvented by setting index_prefix.

bin/py-mysql2pgsql

@@ -22,6 +22,12 @@ if __name__ == '__main__':
         default='mysql2pgsql.yml',
         help='Location of configuration file (default: %(default)s). If none exists at that path, one will be created for you.',
         )
+    parser.add_argument(
+        '-s', '--single-transaction',
+        default=False,
+        action='store_true',
+        help='Make data migration in signle transaction (default: False)',
+        )
     parser.add_argument(
         '-V', '--version',
         action='store_true',

mysql2pgsql/lib/postgres_db_writer.py

@@ -28,11 +28,12 @@ class FileObjFaker(object):
           - `processor`:
           - `verbose`: whether or not to log progress to :py:obj:`stdout`
         """
-        def __init__(self, table, data, processor, verbose=False):
+        def __init__(self, table, data, processor, verbose=False, single_transaction=False):
             self.data = iter(data)
             self.table = table
             self.processor = processor
             self.verbose = verbose
+            self.single_transaction = single_transaction
 
             if verbose:
                 self.idx = 1
@@ -68,9 +69,10 @@ def readline(self, *args, **kwargs):
         def read(self, *args, **kwargs):
             return self.readline(*args, **kwargs)
 
-    def __init__(self, db_options, verbose=False, *args, **kwargs):
+    def __init__(self, db_options, verbose=False, single_transaction=False, *args, **kwargs):
         super(PostgresDbWriter, self).__init__(*args, **kwargs)
         self.verbose = verbose
+        self.single_transaction = single_transaction
         self.db_options = {
             'host': str(db_options['hostname']),
             'port': db_options.get('port', 5432),
@@ -107,7 +109,8 @@ def execute(self, sql, args=(), many=False):
                 cur.executemany(sql, args)
             else:
                 cur.execute(sql, args)
-            self.conn.commit()
+            if not self.single_transaction:
+                self.conn.commit()
 
     def copy_from(self, file_obj, table_name, columns):
         with closing(self.conn.cursor()) as cur:
@@ -116,10 +119,13 @@ def copy_from(self, file_obj, table_name, columns):
                           columns=columns
                           )
 
-        self.conn.commit()
+        if not self.single_transaction:
+            self.conn.commit()
 
     def close(self):
         """Closes connection to the PostgreSQL server"""
+        if self.single_transaction:
+            self.conn.commit()
         self.conn.close()
 
     def exists(self, relname):

mysql2pgsql/mysql2pgsql.py

@@ -25,12 +25,13 @@ def convert(self):
 
         if self.file_options['destination']['file']:
             writer = PostgresFileWriter(self._get_file(self.file_options['destination']['file']), 
-                                        self.run_options.verbose, 
+                                        self.run_options.verbose,
                                         tz=self.file_options.get('timezone', False),
                                         index_prefix=self.file_options.get("index_prefix", ''))
         else:
             writer = PostgresDbWriter(self.file_options['destination']['postgres'], 
-                                      self.run_options.verbose, 
+                                      self.run_options.verbose,
+                                      self.run_options.single_transaction,
                                       tz=self.file_options.get('timezone', False),
                                       index_prefix=self.file_options.get("index_prefix", ''))