From de1fc76c5bc4a9fd72ea2172ab1a54c6e16a34cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20M=2E=20Basto?= Date: Tue, 12 Feb 2019 01:07:05 +0000 Subject: [PATCH 1/9] Shows errors if they happen else nothing change. I got errors on tar -cf like tar: ./objects/8e/38dd9c4ccf3b3216c2e89b2e7e4f6d5864e787: file changed as we read it tar: ./objects/8e/c741bf8ef59ef5299f0fa765af6463a5a277aa: File removed before we read it tar: ./objects/8e/8ac444704cd861f38b1ebfa2045d706cbdf0fc: File removed before we read it tar: ./objects/8e/d68e10c4f4a2296dfefc4b699a164fee6d62b4: File removed before we read it tar: ./objects/8e/38a0fbbf7bff29dedb3123eeea435996b3e1bd: File removed before we read it tar: ./objects/8e: file changed as we read it tar: ./objects/0d: File removed before we read it tar: ./objects/0e: File removed before we read it tar: ./objects/73: File removed before we read it --- svn2github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/svn2github.py b/svn2github.py index 97ddc0e..5aca930 100755 --- a/svn2github.py +++ b/svn2github.py @@ -98,7 +98,7 @@ def unpack_cache(cache_path, git_dir): def save_cache(cache_path, tmp_path, git_dir): dot_git_dir = os.path.join(git_dir, ".git") - proc.run(["tar", "-cf", tmp_path, "."], check=True, cwd=dot_git_dir, stderr=DEVNULL, stdin=DEVNULL, stdout=DEVNULL) + proc.run(["tar", "-cf", tmp_path, "."], check=True, cwd=dot_git_dir) shutil.copyfile(tmp_path, cache_path) From 9576dd0ed1c37e41eab0942392ad374dc24dcb36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20M=2E=20Basto?= Date: Tue, 12 Feb 2019 01:12:04 +0000 Subject: [PATCH 2/9] Sleep 60 seconds before start archive fix the issue for me --- svn2github.py | 1 + 1 file changed, 1 insertion(+) diff --git a/svn2github.py b/svn2github.py index 5aca930..a406555 100755 --- a/svn2github.py +++ b/svn2github.py @@ -98,6 +98,7 @@ def unpack_cache(cache_path, git_dir): def save_cache(cache_path, tmp_path, git_dir): dot_git_dir = os.path.join(git_dir, ".git") + proc.run(["sleep", "60"], check=True, cwd=dot_git_dir) proc.run(["tar", "-cf", tmp_path, "."], check=True, cwd=dot_git_dir) shutil.copyfile(tmp_path, cache_path) From 61ac4e47021bed12c4f398d106a772de0edcadfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20M=2E=20Basto?= Date: Tue, 12 Feb 2019 01:18:03 +0000 Subject: [PATCH 3/9] Why not use XZ compressed data --- svn2github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/svn2github.py b/svn2github.py index a406555..f94ff79 100755 --- a/svn2github.py +++ b/svn2github.py @@ -99,7 +99,7 @@ def unpack_cache(cache_path, git_dir): def save_cache(cache_path, tmp_path, git_dir): dot_git_dir = os.path.join(git_dir, ".git") proc.run(["sleep", "60"], check=True, cwd=dot_git_dir) - proc.run(["tar", "-cf", tmp_path, "."], check=True, cwd=dot_git_dir) + proc.run(["tar", "-cJf", tmp_path, "."], check=True, cwd=dot_git_dir) shutil.copyfile(tmp_path, cache_path) From af16bcc35bdcb90c24cf6ecd51597bdd4fbd233c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20M=2E=20Basto?= Date: Fri, 31 May 2019 19:42:16 +0100 Subject: [PATCH 4/9] Why not use XZ compressed data (part II) --- svn2github.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/svn2github.py b/svn2github.py index f94ff79..6be43c7 100755 --- a/svn2github.py +++ b/svn2github.py @@ -106,7 +106,10 @@ def save_cache(cache_path, tmp_path, git_dir): def sync_github_mirror(github_repo, cache_dir, new_svn_url=None): if cache_dir: os.makedirs(cache_dir, exist_ok=True) - cache_path = os.path.join(cache_dir, "cache." + github_repo.replace("/", ".") + ".tar") + cache_old_path = os.path.join(cache_dir, "cache." + github_repo.replace("/", ".") + ".tar") + cache_path = os.path.join(cache_dir, "cache." + github_repo.replace("/", ".") + ".tar.xz") + if os.path.exists(cache_old_path): + shutil.move(cache_old_path, cache_path) cached = os.path.exists(cache_path) else: cached = False @@ -154,7 +157,7 @@ def sync_github_mirror(github_repo, cache_dir, new_svn_url=None): if cache_dir: print("Saving Git directory to cache") - save_cache(cache_path, os.path.join(tmp_dir, "cache.tar"), git_dir) + save_cache(cache_path, os.path.join(tmp_dir, "cache.tar.xz"), git_dir) def main(): From 5952d037a9463feebdf5118d89d894caed775a7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20M=2E=20Basto?= Date: Fri, 31 May 2019 19:52:33 +0100 Subject: [PATCH 5/9] Add argument --git-dir set a local dir without be compress/decompress --- svn2github.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/svn2github.py b/svn2github.py index 6be43c7..1013873 100755 --- a/svn2github.py +++ b/svn2github.py @@ -103,7 +103,7 @@ def save_cache(cache_path, tmp_path, git_dir): shutil.copyfile(tmp_path, cache_path) -def sync_github_mirror(github_repo, cache_dir, new_svn_url=None): +def sync_github_mirror(github_repo, cache_dir, new_svn_url=None, new_git_dir=None): if cache_dir: os.makedirs(cache_dir, exist_ok=True) cache_old_path = os.path.join(cache_dir, "cache." + github_repo.replace("/", ".") + ".tar") @@ -116,12 +116,17 @@ def sync_github_mirror(github_repo, cache_dir, new_svn_url=None): github_url = "git@github.com:" + github_repo + ".git" - with tempfile.TemporaryDirectory(prefix="svn2github-") as tmp_dir: + if not new_git_dir: + tmp_dir = os.path.join(tempfile.TemporaryDirectory(prefix="svn2github-"), "repo") + else: + tmp_dir = new_git_dir + + if tmp_dir: git_dir = os.path.join(tmp_dir, "repo") if cached and not new_svn_url: print("Using cached Git repository from " + cache_path) unpack_cache(cache_path, git_dir) - else: + elif not new_git_dir or not os.path.exists(new_git_dir): print("Cloning " + github_url) git_clone(github_url, git_dir) @@ -131,6 +136,7 @@ def sync_github_mirror(github_repo, cache_dir, new_svn_url=None): git_svn_info = GitSvnInfo(svn_url=new_svn_url, svn_revision=0, svn_uuid=None) else: git_svn_info = get_svn_info_from_git(git_dir) + print("svn_url = %s, svn_revison = %s" % (git_svn_info.svn_url, git_svn_info.svn_revision)) print("Checking for SVN updates") upstream_revision = get_last_revision_from_svn(git_svn_info.svn_url) @@ -163,6 +169,7 @@ def sync_github_mirror(github_repo, cache_dir, new_svn_url=None): def main(): parser = argparse.ArgumentParser(description="Mirror SVN repositories to GitHub") parser.add_argument("--cache-dir", help="Directory to keep the cached data to avoid re-downloading all SVN and Git history each time. This is optional, but highly recommended") + parser.add_argument("--git-dir", help="git local dir without be compressed") subparsers = parser.add_subparsers() subparser_import = subparsers.add_parser("import", help="Import SVN repository to the GitHub repo") @@ -174,7 +181,8 @@ def main(): args = parser.parse_args(sys.argv[1:] or ["--help"]) new_svn_url = args.svn_url if "svn_url" in args else None - sync_github_mirror(args.github_repo, args.cache_dir, new_svn_url=new_svn_url) + new_git_dir = args.git_dir if "git_dir" in args else None + sync_github_mirror(args.github_repo, args.cache_dir, new_svn_url=new_svn_url, new_git_dir=new_git_dir) From 936adb3b3a689f3721b85eec0a32c1c68ec25204 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20M=2E=20Basto?= Date: Fri, 31 May 2019 23:28:33 +0100 Subject: [PATCH 6/9] Update README.md --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index 4e23db9..5679ca7 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,12 @@ optional arguments: Directory to keep the cached data to avoid re- downloading all SVN and Git history each time. This is optional, but highly recommended + --git-dir GIT_DIR + Directory to keep the cached data to avoid re- + downloading all SVN and Git history each time. This is + optional, but highly recommended + the difference for cache-dir is that is not compress and avoid decompress, copy all to an temporary directory and compress again. + ==== @@ -79,3 +85,13 @@ optional arguments: -h, --help show this help message and exit ``` +## Examples and migration from cache-dir to git-dir + +### 1 - Before the existence of --git-dir option +./svn2github.py --cache-dir /home/sergio/rpmfusion/new/VirtualBox/svn2github/virtualbox update sergiomb2/virtualbox + +### 2 - Migration from --cache-dir to --git-dir option (just run one time) +./svn2github.py --git-dir /home/sergio/rpmfusion/new/VirtualBox/svn2github/virtualbox-repo/ --cache-dir /home/sergio/rpmfusion/new/VirtualBox/svn2github/virtualbox update sergiomb2/virtualbox + +#### 3 - Now we may delete cache_dir and run option just with --git-dir option +./svn2github.py --git-dir /home/sergio/rpmfusion/new/VirtualBox/svn2github/virtualbox-repo/ update sergiomb2/virtualbox From 7c7061708acc0e9ec7f2bbe20e8a8c5a178888d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20M=2E=20Basto?= Date: Tue, 26 Nov 2019 20:06:57 +0000 Subject: [PATCH 7/9] Bug fix and raise some errors instead ignore it. --- svn2github.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/svn2github.py b/svn2github.py index 1013873..b583772 100755 --- a/svn2github.py +++ b/svn2github.py @@ -28,11 +28,11 @@ def get_last_revision_from_svn(svn_url): if m: return int(m.group(1)) - return Svn2GithubException("svn info {} output did not specify the current revision".format(svn_url)) + raise Svn2GithubException("svn info {} output did not specify the current revision".format(svn_url)) def run_git_cmd(args, git_dir): - return proc.run(["git"] + args, check=True, cwd=git_dir, stderr=DEVNULL, stdin=DEVNULL, stdout=PIPE) + return proc.run(["git"] + args, check=True, cwd=git_dir, stdout=PIPE) def is_repo_empty(git_dir): @@ -50,7 +50,8 @@ def get_svn_info_from_git(git_dir): if m: return GitSvnInfo(svn_url=m.group(1), svn_revision=int(m.group(2)), svn_uuid=m.group(3)) - return Svn2GithubException("git log -1 HEAD --pretty=%b output did not specify the current revision") + print(result) + raise Svn2GithubException("git log -1 HEAD --pretty=%b output did not specify the current revision") def git_svn_init(git_svn_info, git_dir): @@ -136,6 +137,7 @@ def sync_github_mirror(github_repo, cache_dir, new_svn_url=None, new_git_dir=Non git_svn_info = GitSvnInfo(svn_url=new_svn_url, svn_revision=0, svn_uuid=None) else: git_svn_info = get_svn_info_from_git(git_dir) + #print(git_svn_info) print("svn_url = %s, svn_revison = %s" % (git_svn_info.svn_url, git_svn_info.svn_revision)) print("Checking for SVN updates") @@ -148,7 +150,7 @@ def sync_github_mirror(github_repo, cache_dir, new_svn_url=None, new_git_dir=Non return print("Fetching from SVN", end="") - if not cached or new_svn_url: + if (not cached or new_svn_url) and not new_git_dir: git_svn_init(git_svn_info, git_dir) for rev in git_svn_fetch(git_dir): From 5d54bde76a7c36f4bc169bc406ca8e71e226c11e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20M=2E=20Basto?= Date: Mon, 2 Aug 2021 03:05:13 +0100 Subject: [PATCH 8/9] fix bug --- svn2github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/svn2github.py b/svn2github.py index b583772..2df67be 100755 --- a/svn2github.py +++ b/svn2github.py @@ -118,7 +118,7 @@ def sync_github_mirror(github_repo, cache_dir, new_svn_url=None, new_git_dir=Non github_url = "git@github.com:" + github_repo + ".git" if not new_git_dir: - tmp_dir = os.path.join(tempfile.TemporaryDirectory(prefix="svn2github-"), "repo") + tmp_dir = tempfile.TemporaryDirectory(prefix="svn2github-").name else: tmp_dir = new_git_dir From 715966315e3d4fed95497a72690fa1eafd21500c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20M=2E=20Basto?= Date: Mon, 2 Aug 2021 13:26:51 +0100 Subject: [PATCH 9/9] bug fix --- svn2github.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/svn2github.py b/svn2github.py index 2df67be..0cda1a9 100755 --- a/svn2github.py +++ b/svn2github.py @@ -127,7 +127,9 @@ def sync_github_mirror(github_repo, cache_dir, new_svn_url=None, new_git_dir=Non if cached and not new_svn_url: print("Using cached Git repository from " + cache_path) unpack_cache(cache_path, git_dir) - elif not new_git_dir or not os.path.exists(new_git_dir): + elif new_git_dir and os.path.exists(new_git_dir): + pass + else: print("Cloning " + github_url) git_clone(github_url, git_dir)