From 0c39df4af2cf9a909943a5c753ed0ddc0b4d63b3 Mon Sep 17 00:00:00 2001 From: Gianni Furger Date: Tue, 29 Aug 2017 13:22:47 +0200 Subject: [PATCH 01/18] updated docs --- README.markdown | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/README.markdown b/README.markdown index daff30a..44af626 100644 --- a/README.markdown +++ b/README.markdown @@ -22,7 +22,7 @@ Installation is simple with Usage ----- ```` -usage: github-backup.py [-h] [-c] [-m] [-g ARGS] [-s SUFFIX] +usage: github-backup.py [-h] [-c] [-m] [-f] [-g ARGS] [-o ORGANIZATION] [-p PASSWORD] [-P PREFIX] [-s SUFFIX] [-S] [-t TOKEN] username backupdir makes a backup of all of a github user's repositories @@ -32,13 +32,23 @@ positional arguments: backupdir The folder where you want your backups to go optional arguments: - -h, --help show this help message and exit + -h, --help Show this help message and exit -c, --cron Use this when running from a cron job -m, --mirror Create a bare mirror -f, --skip-forks Skip forks -g ARGS, --git ARGS Pass extra arguments to git + -o ORGANIZATION, --organization ORGANIZATION + Backup Organizational repositories + -p PASSWORD, --password PASSWORD + Authenticate with Github API + -P PREFIX, --prefix PREFIX + Add prefix to repository directory names -s SUFFIX, --suffix SUFFIX Add suffix to repository directory names + -S, --ssh Use SSH protocol + -t TOKEN, --token TOKEN + Authenticate with Github API using OAuth token + ```` Then, put it in a cron job somewhere and forget about it for eternity. From 534776402af8712923ad87ac716d87d6dc3639e3 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Fri, 25 Oct 2013 10:58:22 +0200 Subject: [PATCH 02/18] added support for oauth token and fixed little bug in pygithub3 configuration --- github-backup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/github-backup.py b/github-backup.py index 5e5923b..a93833e 100755 --- a/github-backup.py +++ b/github-backup.py @@ -23,6 +23,11 @@ def main(): # Make the connection to Github here. config = { 'user': args.username } + args.backupdir = args.backupdir.rstrip("/") + + if (args.token): + config['token'] = args.token + if (args.password): config['password'] = args.password config['login'] = args.username @@ -39,7 +44,7 @@ def main(): if args.organization: user_repos = gh.repos.list_by_org(args.organization).all() else: - user_repos = gh.repos.list().all() + user_repos = gh.repos.list(user=args.username).all() for repo in user_repos: repo.user = gh.users.get(repo.owner.login) From 4d4daaa0fff3dfa7a0782e076bf466556c9dfe5d Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Thu, 14 Nov 2013 18:56:12 +0100 Subject: [PATCH 03/18] replaced os.system() by subprocess.call() and fixed escaping errors --- github-backup.py | 62 ++++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/github-backup.py b/github-backup.py index a93833e..8167016 100755 --- a/github-backup.py +++ b/github-backup.py @@ -9,16 +9,19 @@ Created: Fri Jun 15 2012 from pygithub3 import Github from argparse import ArgumentParser +import subprocess import os - def main(): parser = init_parser() args = parser.parse_args() + if not args.git: + args.git = [] + # Process args if args.cron: - args.git += "--quiet" + args.git.append("--quiet") # Make the connection to Github here. config = { 'user': args.username } @@ -62,10 +65,10 @@ def init_parser(): parser.add_argument("username", help="A Github username") parser.add_argument("backupdir", help="The folder where you want your backups to go") - parser.add_argument("-c","--cron", help="Use this when running from a cron job", action="store_true") - parser.add_argument("-m","--mirror", help="Create a bare mirror", action="store_true") - parser.add_argument("-f","--skip-forks", help="Skip forks", action="store_true") - parser.add_argument("-g","--git", help="Pass extra arguments to git", default="", metavar="ARGS") + parser.add_argument("-c", "--cron", help="Use this when running from a cron job", action="store_true") + parser.add_argument("-m", "--mirror", help="Create a bare mirror", action="store_true") + parser.add_argument("-f", "--skip-forks", help="Skip forks", action="store_true") + parser.add_argument("-g", "--git", nargs="+", help="Pass extra arguments to git", default="", metavar="ARGS") parser.add_argument("-s", "--suffix", help="Add suffix to repository directory names", default="") parser.add_argument("-p", "--password", help="Authenticate with Github API") parser.add_argument("-P","--prefix", help="Add prefix to repository directory names", default="") @@ -92,41 +95,42 @@ def process_repo(repo, args): def clone_repo(repo, dir, args): + url = repo.ssh_url if args.ssh else repo.git_url + if args.mirror: - options = args.git + " --mirror" + git("clone", ["--mirror", url, dir], args.git, dir) else: - options = args.git - - os.system('git clone %s %s %s'%(options, repo.ssh_url if args.ssh else repo.git_url, dir)) - + git("clone", [url, dir], args.git, dir) def update_repo(repo, dir, args): - savedPath = os.getcwd() - os.chdir(dir) - # GitHub => Local # TODO: use subprocess package and fork git into background (major performance boost expected) if args.mirror: - os.system("git fetch %s"%(args.git + " --prune",)) + git("fetch", ["--prune"], args.git, dir) else: - os.system("git pull %s"%(args.git,)) + git("pull", gargs=args.git, gdir=dir) # Fetch description and owner (useful for gitweb, cgit etc.) - # TODO: can we combine that in a single call to 'git config' - if repo.description is not None: - os.system("git config --local gitweb.description %s"%(shell_escape(repo.description),)) - if repo.user.name is not None and repo.user.email is not None: - os.system("git config --local gitweb.owner %s"%(shell_escape("%s <%s>"%(repo.user.name, repo.user.email.encode("utf-8"))),)) + if repo.description: + git("config", ["--local", "gitweb.description", repo.description], gdir=dir) + if repo.user.name and repo.user.email: + git("config", ["--local", "gitweb.owner", "%s <%s>" %(repo.user.name, repo.user.email.encode("utf-8"))], gdir=dir) + git("config", ["--local", "cgit.name", repo.name], gdir=dir) + git("config", ["--local", "cgit.defbranch", repo.default_branch], gdir=dir) + git("config", ["--local", "cgit.clone-url", repo.clone_url], gdir=dir) - os.system("git config --local cgit.name %s"%(shell_escape(repo.name),)) - os.system("git config --local cgit.defbranch %s"%(shell_escape(repo.default_branch),)) - os.system("git config --local cgit.clone-url %s"%(shell_escape(repo.clone_url),)) - - os.chdir(savedPath) -def shell_escape(str): - if str: - return "'" + unicode(str.replace("'", "'\\''")).encode("utf-8") + "'" +def git(gcmd, args=[], gargs=[], gdir=""): + cmd = ["git"] + if gdir: + cmd.append("--git-dir") + cmd.append(gdir) + cmd.append(gcmd) + cmd.extend(gargs) + cmd.extend(args) + + subprocess.call(cmd) +>>>>>>> replaced os.system() by subprocess.call() and fixed escaping errors if __name__ == "__main__": main() From 0f5ef8d394cc85cd525a6cf931e1df35468653af Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Thu, 26 Jun 2014 00:14:13 +0200 Subject: [PATCH 04/18] sorted cli options --- github-backup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/github-backup.py b/github-backup.py index 8167016..a6405d0 100755 --- a/github-backup.py +++ b/github-backup.py @@ -69,11 +69,12 @@ def init_parser(): parser.add_argument("-m", "--mirror", help="Create a bare mirror", action="store_true") parser.add_argument("-f", "--skip-forks", help="Skip forks", action="store_true") parser.add_argument("-g", "--git", nargs="+", help="Pass extra arguments to git", default="", metavar="ARGS") + parser.add_argument("-S", "--ssh", help="Use SSH protocol", action="store_true") parser.add_argument("-s", "--suffix", help="Add suffix to repository directory names", default="") + parser.add_argument("-P", "--prefix", help="Add prefix to repository directory names", default="") parser.add_argument("-p", "--password", help="Authenticate with Github API") parser.add_argument("-P","--prefix", help="Add prefix to repository directory names", default="") - parser.add_argument("-o","--organization", help="Backup Organizational repositories") - parser.add_argument("-S","--ssh", help="Use SSH protocol", action="store_true") + parser.add_argument("-o", "--organization", help="Backup Organizational repositories", metavar="ORG") parser.add_argument("-t","--token", help="Authenticate with Github API using OAuth token", default="") return parser From fb7a4cab043a0cf0a116ee7b9413f358ffa45838 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Thu, 26 Jun 2014 00:14:25 +0200 Subject: [PATCH 05/18] cosmetic changes --- github-backup.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/github-backup.py b/github-backup.py index a6405d0..ba80370 100755 --- a/github-backup.py +++ b/github-backup.py @@ -91,17 +91,17 @@ def process_repo(repo, args): clone_repo(repo, dir, args) else: if not args.cron: print("Repo already exists, let's try to update it instead") - - update_repo(repo, dir, args) + update_repo(repo, dir, args) def clone_repo(repo, dir, args): - url = repo.ssh_url if args.ssh else repo.git_url + params = [repo.repo.ssh_url if args.ssh else repo.git_url, dir] if args.mirror: - git("clone", ["--mirror", url, dir], args.git, dir) - else: - git("clone", [url, dir], args.git, dir) + params.insert(0, ["--mirror"]) + + git("clone", params, args.git, dir) + def update_repo(repo, dir, args): # GitHub => Local @@ -126,12 +126,12 @@ def git(gcmd, args=[], gargs=[], gdir=""): if gdir: cmd.append("--git-dir") cmd.append(gdir) + cmd.append(gcmd) cmd.extend(gargs) cmd.extend(args) subprocess.call(cmd) ->>>>>>> replaced os.system() by subprocess.call() and fixed escaping errors if __name__ == "__main__": main() From 025d4d41babc06fbd774aa96776991ab2d684d40 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Fri, 25 Oct 2013 10:58:22 +0200 Subject: [PATCH 06/18] added support for oauth token and fixed little bug in pygithub3 configuration Conflicts: github-backup.py --- github-backup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/github-backup.py b/github-backup.py index ba80370..4cb70c4 100755 --- a/github-backup.py +++ b/github-backup.py @@ -73,9 +73,9 @@ def init_parser(): parser.add_argument("-s", "--suffix", help="Add suffix to repository directory names", default="") parser.add_argument("-P", "--prefix", help="Add prefix to repository directory names", default="") parser.add_argument("-p", "--password", help="Authenticate with Github API") - parser.add_argument("-P","--prefix", help="Add prefix to repository directory names", default="") + parser.add_argument("-P", "--prefix", help="Add prefix to repository directory names", default="") + parser.add_argument("-t", "--token", help="Authenticate with Github API using OAuth token", default="") parser.add_argument("-o", "--organization", help="Backup Organizational repositories", metavar="ORG") - parser.add_argument("-t","--token", help="Authenticate with Github API using OAuth token", default="") return parser From 202b9e44a8abb1dc3bde198705e97a60ff406909 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Thu, 26 Jun 2014 00:28:35 +0200 Subject: [PATCH 07/18] updated README --- README.markdown | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/README.markdown b/README.markdown index daff30a..2245017 100644 --- a/README.markdown +++ b/README.markdown @@ -1,17 +1,13 @@ GitHub-Backup -============= - -Idea/original implementation by Chris Lockfort (clockfort@csh.rit.edu) (Github username: Clockfort) - -Python version by Anthony Gargiulo (anthony@agargiulo.com) (Github username: agargiulo) +============================ Description ----- +---------------------------- -GitHub-Backup makes a local backup copy of all of a github user's (or github organization's) repositories. +GitHub-Backup makes a local backup copy of all of a github user's (or github organization's) repositories. Dependencies ----- +---------------------------- GitHub-Backup requires `pygithub3` a Python library for the GitHub API v3. @@ -20,9 +16,10 @@ Installation is simple with pip install pygithub3 Usage ------ +---------------------------- ```` -usage: github-backup.py [-h] [-c] [-m] [-g ARGS] [-s SUFFIX] +usage: github-backup.py [-h] [-c] [-m] [-S] [-g ARGS] [-s SUFFIX] [-P PREFIX] + [-p PASSWORD] [-t TOKEN] [-o ORG] username backupdir makes a backup of all of a github user's repositories @@ -36,9 +33,18 @@ optional arguments: -c, --cron Use this when running from a cron job -m, --mirror Create a bare mirror -f, --skip-forks Skip forks + -S, --ssh Use SSH protocol -g ARGS, --git ARGS Pass extra arguments to git -s SUFFIX, --suffix SUFFIX Add suffix to repository directory names + -P PREFIX, --prefix PREFIX + Add prefix to repository directory names + -p PASSWORD, --password PASSWORD + Authenticate with Github API + -t TOKEN, --token TOKEN + OAuth token for authentification + -o ORG, --organization ORG + Backup Organizational repositories ```` Then, put it in a cron job somewhere and forget about it for eternity. @@ -62,3 +68,17 @@ Questions, Improvements, Etc ----------------------------- If you have any improvements, I'm happy, (grateful, in fact) to entertain pull requests/patches, just drop me a line or message me on GitHub. + +Contributors +---------------------------- + +Idea/original implementation by + +- Chris Lockfort (clockfort@csh.rit.edu) (Github: Clockfort) + (Original idea) + +- Anthony Gargiulo (anthony@agargiulo.com) (Github: agargiulo) + (Python version) + +- Steffen Vogel (post@steffenvogel.de) (Github: stv0g) + (A lot of patches and improvements) From abd1fa5195ed6f7f847e61384b939cdcb0183d9e Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sat, 2 Feb 2019 21:27:20 +0100 Subject: [PATCH 08/18] several encoding fixes and more --- github-backup.py | 52 +++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/github-backup.py b/github-backup.py index 4cb70c4..2199df0 100755 --- a/github-backup.py +++ b/github-backup.py @@ -43,17 +43,16 @@ def main(): gh = Github(**config) - # Get all of the given user's repos - if args.organization: - user_repos = gh.repos.list_by_org(args.organization).all() - else: - user_repos = gh.repos.list(user=args.username).all() - + # Get all repos + users = { } + user_repos = gh.repos.list(user=args.username).all() for repo in user_repos: - repo.user = gh.users.get(repo.owner.login) - fullrepo = gh.repos.get(repo.owner.login, repo.name) + if repo.owner.login not in users: + users[repo.owner.login] = gh.users.get(repo.owner.login) + + repo.user = users[repo.owner.login] if not (args.skip_forks and hasattr(fullrepo, 'parent') and hasattr(fullrepo, 'source')): - process_repo(repo, args) + process_repo(repo, args) def init_parser(): @@ -71,7 +70,6 @@ def init_parser(): parser.add_argument("-g", "--git", nargs="+", help="Pass extra arguments to git", default="", metavar="ARGS") parser.add_argument("-S", "--ssh", help="Use SSH protocol", action="store_true") parser.add_argument("-s", "--suffix", help="Add suffix to repository directory names", default="") - parser.add_argument("-P", "--prefix", help="Add prefix to repository directory names", default="") parser.add_argument("-p", "--password", help="Authenticate with Github API") parser.add_argument("-P", "--prefix", help="Add prefix to repository directory names", default="") parser.add_argument("-t", "--token", help="Authenticate with Github API using OAuth token", default="") @@ -83,24 +81,26 @@ def process_repo(repo, args): if not args.cron: print("Processing repo: %s"%(repo.full_name)) - dir = "%s/%s"%(args.backupdir, args.prefix + repo.name + args.suffix) - config = "%s/%s"%(dir, "config" if args.mirror else ".git/config") + dir = "%s/%s" % (args.backupdir, repo.name + args.suffix) + config = "%s/%s" % (dir, "config" if args.mirror else ".git/config") if not os.access(config, os.F_OK): - if not args.cron: print("Repo doesn't exists, lets clone it") + if not args.cron: + print("Repo doesn't exists, lets clone it") + clone_repo(repo, dir, args) else: - if not args.cron: print("Repo already exists, let's try to update it instead") - update_repo(repo, dir, args) + if not args.cron: + print("Repo already exists, let's try to update it instead") + + update_repo(repo, dir, args) def clone_repo(repo, dir, args): - params = [repo.repo.ssh_url if args.ssh else repo.git_url, dir] - if args.mirror: - params.insert(0, ["--mirror"]) - - git("clone", params, args.git, dir) + git("clone", ["--mirror", repo.git_url, dir], args.git, dir) + else: + git("clone", [repo.git_url, dir], args.git, dir) def update_repo(repo, dir, args): @@ -113,20 +113,18 @@ def update_repo(repo, dir, args): # Fetch description and owner (useful for gitweb, cgit etc.) if repo.description: - git("config", ["--local", "gitweb.description", repo.description], gdir=dir) + git("config", ["--local", "gitweb.description", repo.description.encode("utf-8")], gdir=dir) if repo.user.name and repo.user.email: - git("config", ["--local", "gitweb.owner", "%s <%s>" %(repo.user.name, repo.user.email.encode("utf-8"))], gdir=dir) - git("config", ["--local", "cgit.name", repo.name], gdir=dir) - git("config", ["--local", "cgit.defbranch", repo.default_branch], gdir=dir) - git("config", ["--local", "cgit.clone-url", repo.clone_url], gdir=dir) - + git("config", ["--local", "gitweb.owner", "%s <%s>" % (repo.user.name.encode("utf-8"), repo.user.email.encode("utf-8"))], gdir=dir) + git("config", ["--local", "cgit.name", str(repo.name)], gdir=dir) + git("config", ["--local", "cgit.defbranch", str(repo.default_branch)], gdir=dir) + git("config", ["--local", "cgit.clone-url", str(repo.clone_url)], gdir=dir) def git(gcmd, args=[], gargs=[], gdir=""): cmd = ["git"] if gdir: cmd.append("--git-dir") cmd.append(gdir) - cmd.append(gcmd) cmd.extend(gargs) cmd.extend(args) From 92db7a84cfb3af6e40e0bf4cc515ef8036b24370 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 5 May 2019 01:48:21 +0200 Subject: [PATCH 09/18] pep8: use tabs for indention --- github-backup.py | 172 +++++++++++++++++++++++------------------------ 1 file changed, 86 insertions(+), 86 deletions(-) diff --git a/github-backup.py b/github-backup.py index 2199df0..460c08e 100755 --- a/github-backup.py +++ b/github-backup.py @@ -13,123 +13,123 @@ import subprocess import os def main(): - parser = init_parser() - args = parser.parse_args() + parser = init_parser() + args = parser.parse_args() - if not args.git: - args.git = [] + if not args.git: + args.git = [] - # Process args - if args.cron: - args.git.append("--quiet") + # Process args + if args.cron: + args.git.append("--quiet") - # Make the connection to Github here. - config = { 'user': args.username } + # Make the connection to Github here. + config = { 'user': args.username } - args.backupdir = args.backupdir.rstrip("/") + args.backupdir = args.backupdir.rstrip("/") - if (args.token): - config['token'] = args.token + if (args.token): + config['token'] = args.token - if (args.password): - config['password'] = args.password - config['login'] = args.username + if (args.password): + config['password'] = args.password + config['login'] = args.username - # if both password and token are specified, the token will be - # used, according to pygithub3 sources - # however, the username isn't required when using a token - if (args.token): - config['token'] = args.token + # if both password and token are specified, the token will be + # used, according to pygithub3 sources + # however, the username isn't required when using a token + if (args.token): + config['token'] = args.token - gh = Github(**config) + gh = Github(**config) - # Get all repos - users = { } - user_repos = gh.repos.list(user=args.username).all() - for repo in user_repos: - if repo.owner.login not in users: - users[repo.owner.login] = gh.users.get(repo.owner.login) + # Get all repos + users = { } + user_repos = gh.repos.list(user=args.username).all() + for repo in user_repos: + if repo.owner.login not in users: + users[repo.owner.login] = gh.users.get(repo.owner.login) - repo.user = users[repo.owner.login] - if not (args.skip_forks and hasattr(fullrepo, 'parent') and hasattr(fullrepo, 'source')): - process_repo(repo, args) + repo.user = users[repo.owner.login] + if not (args.skip_forks and hasattr(fullrepo, 'parent') and hasattr(fullrepo, 'source')): + process_repo(repo, args) def init_parser(): - """ - set up the argument parser - """ + """ + set up the argument parser + """ - parser = ArgumentParser(description="makes a backup of all of a github user's repositories") + parser = ArgumentParser(description="makes a backup of all of a github user's repositories") - parser.add_argument("username", help="A Github username") - parser.add_argument("backupdir", help="The folder where you want your backups to go") - parser.add_argument("-c", "--cron", help="Use this when running from a cron job", action="store_true") - parser.add_argument("-m", "--mirror", help="Create a bare mirror", action="store_true") - parser.add_argument("-f", "--skip-forks", help="Skip forks", action="store_true") - parser.add_argument("-g", "--git", nargs="+", help="Pass extra arguments to git", default="", metavar="ARGS") - parser.add_argument("-S", "--ssh", help="Use SSH protocol", action="store_true") - parser.add_argument("-s", "--suffix", help="Add suffix to repository directory names", default="") - parser.add_argument("-p", "--password", help="Authenticate with Github API") - parser.add_argument("-P", "--prefix", help="Add prefix to repository directory names", default="") - parser.add_argument("-t", "--token", help="Authenticate with Github API using OAuth token", default="") - parser.add_argument("-o", "--organization", help="Backup Organizational repositories", metavar="ORG") + parser.add_argument("username", help="A Github username") + parser.add_argument("backupdir", help="The folder where you want your backups to go") + parser.add_argument("-c", "--cron", help="Use this when running from a cron job", action="store_true") + parser.add_argument("-m", "--mirror", help="Create a bare mirror", action="store_true") + parser.add_argument("-f", "--skip-forks", help="Skip forks", action="store_true") + parser.add_argument("-g", "--git", nargs="+", help="Pass extra arguments to git", default="", metavar="ARGS") + parser.add_argument("-S", "--ssh", help="Use SSH protocol", action="store_true") + parser.add_argument("-s", "--suffix", help="Add suffix to repository directory names", default="") + parser.add_argument("-p", "--password", help="Authenticate with Github API") + parser.add_argument("-P", "--prefix", help="Add prefix to repository directory names", default="") + parser.add_argument("-t", "--token", help="Authenticate with Github API using OAuth token", default="") + parser.add_argument("-o", "--organization", help="Backup Organizational repositories", metavar="ORG") - return parser + return parser def process_repo(repo, args): - if not args.cron: - print("Processing repo: %s"%(repo.full_name)) + if not args.cron: + print("Processing repo: %s"%(repo.full_name)) - dir = "%s/%s" % (args.backupdir, repo.name + args.suffix) - config = "%s/%s" % (dir, "config" if args.mirror else ".git/config") + dir = "%s/%s" % (args.backupdir, repo.name + args.suffix) + config = "%s/%s" % (dir, "config" if args.mirror else ".git/config") - if not os.access(config, os.F_OK): - if not args.cron: - print("Repo doesn't exists, lets clone it") + if not os.access(config, os.F_OK): + if not args.cron: + print("Repo doesn't exists, lets clone it") - clone_repo(repo, dir, args) - else: - if not args.cron: - print("Repo already exists, let's try to update it instead") + clone_repo(repo, dir, args) + else: + if not args.cron: + print("Repo already exists, let's try to update it instead") - update_repo(repo, dir, args) + update_repo(repo, dir, args) def clone_repo(repo, dir, args): - if args.mirror: - git("clone", ["--mirror", repo.git_url, dir], args.git, dir) - else: - git("clone", [repo.git_url, dir], args.git, dir) + if args.mirror: + git("clone", ["--mirror", repo.git_url, dir], args.git, dir) + else: + git("clone", [repo.git_url, dir], args.git, dir) def update_repo(repo, dir, args): - # GitHub => Local - # TODO: use subprocess package and fork git into background (major performance boost expected) - if args.mirror: - git("fetch", ["--prune"], args.git, dir) - else: - git("pull", gargs=args.git, gdir=dir) + # GitHub => Local + # TODO: use subprocess package and fork git into background (major performance boost expected) + if args.mirror: + git("fetch", ["--prune"], args.git, dir) + else: + git("pull", gargs=args.git, gdir=dir) - # Fetch description and owner (useful for gitweb, cgit etc.) - if repo.description: - git("config", ["--local", "gitweb.description", repo.description.encode("utf-8")], gdir=dir) - if repo.user.name and repo.user.email: - git("config", ["--local", "gitweb.owner", "%s <%s>" % (repo.user.name.encode("utf-8"), repo.user.email.encode("utf-8"))], gdir=dir) - git("config", ["--local", "cgit.name", str(repo.name)], gdir=dir) - git("config", ["--local", "cgit.defbranch", str(repo.default_branch)], gdir=dir) - git("config", ["--local", "cgit.clone-url", str(repo.clone_url)], gdir=dir) + # Fetch description and owner (useful for gitweb, cgit etc.) + if repo.description: + git("config", ["--local", "gitweb.description", repo.description.encode("utf-8")], gdir=dir) + if repo.user.name and repo.user.email: + git("config", ["--local", "gitweb.owner", "%s <%s>" % (repo.user.name.encode("utf-8"), repo.user.email.encode("utf-8"))], gdir=dir) + git("config", ["--local", "cgit.name", str(repo.name)], gdir=dir) + git("config", ["--local", "cgit.defbranch", str(repo.default_branch)], gdir=dir) + git("config", ["--local", "cgit.clone-url", str(repo.clone_url)], gdir=dir) def git(gcmd, args=[], gargs=[], gdir=""): - cmd = ["git"] - if gdir: - cmd.append("--git-dir") - cmd.append(gdir) - cmd.append(gcmd) - cmd.extend(gargs) - cmd.extend(args) + cmd = ["git"] + if gdir: + cmd.append("--git-dir") + cmd.append(gdir) + cmd.append(gcmd) + cmd.extend(gargs) + cmd.extend(args) - subprocess.call(cmd) + subprocess.call(cmd) if __name__ == "__main__": - main() + main() From 37becda672f791a3823e98b06c6a900b36819717 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 5 May 2019 01:56:22 +0200 Subject: [PATCH 10/18] pep8: more code-style fixes --- github-backup.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/github-backup.py b/github-backup.py index 460c08e..3396a1f 100755 --- a/github-backup.py +++ b/github-backup.py @@ -7,11 +7,13 @@ Authors: Anthony Gargiulo (anthony@agargiulo.com) Created: Fri Jun 15 2012 """ + from pygithub3 import Github from argparse import ArgumentParser import subprocess import os + def main(): parser = init_parser() args = parser.parse_args() @@ -24,7 +26,7 @@ def main(): args.git.append("--quiet") # Make the connection to Github here. - config = { 'user': args.username } + config = {'user': args.username} args.backupdir = args.backupdir.rstrip("/") @@ -44,14 +46,16 @@ def main(): gh = Github(**config) # Get all repos - users = { } + users = {} user_repos = gh.repos.list(user=args.username).all() for repo in user_repos: if repo.owner.login not in users: users[repo.owner.login] = gh.users.get(repo.owner.login) repo.user = users[repo.owner.login] - if not (args.skip_forks and hasattr(fullrepo, 'parent') and hasattr(fullrepo, 'source')): + fullrepo = gh.repos.get(repo.owner.login, repo.name) + is_fork = hasattr(fullrepo, 'parent') and hasattr(fullrepo, 'source') + if not (args.skip_forks and is_fork): process_repo(repo, args) @@ -77,9 +81,10 @@ def init_parser(): return parser + def process_repo(repo, args): if not args.cron: - print("Processing repo: %s"%(repo.full_name)) + print("Processing repo: %s" % (repo.full_name)) dir = "%s/%s" % (args.backupdir, repo.name + args.suffix) config = "%s/%s" % (dir, "config" if args.mirror else ".git/config") @@ -105,7 +110,8 @@ def clone_repo(repo, dir, args): def update_repo(repo, dir, args): # GitHub => Local - # TODO: use subprocess package and fork git into background (major performance boost expected) + # TODO: use subprocess package and fork git into + # background (major performance boost expected) if args.mirror: git("fetch", ["--prune"], args.git, dir) else: @@ -113,13 +119,19 @@ def update_repo(repo, dir, args): # Fetch description and owner (useful for gitweb, cgit etc.) if repo.description: - git("config", ["--local", "gitweb.description", repo.description.encode("utf-8")], gdir=dir) + git("config", ["--local", "gitweb.description", + repo.description.encode("utf-8")], gdir=dir) + if repo.user.name and repo.user.email: - git("config", ["--local", "gitweb.owner", "%s <%s>" % (repo.user.name.encode("utf-8"), repo.user.email.encode("utf-8"))], gdir=dir) + owner = "%s <%s>" % (repo.user.name.encode("utf-8"), + repo.user.email.encode("utf-8")) + git("config", ["--local", "gitweb.owner", owner], gdir=dir) + git("config", ["--local", "cgit.name", str(repo.name)], gdir=dir) git("config", ["--local", "cgit.defbranch", str(repo.default_branch)], gdir=dir) git("config", ["--local", "cgit.clone-url", str(repo.clone_url)], gdir=dir) + def git(gcmd, args=[], gargs=[], gdir=""): cmd = ["git"] if gdir: From 7bbbce380c05c28571454fdda420b2cf0d4eed34 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 5 May 2019 02:06:59 +0200 Subject: [PATCH 11/18] added little guide for backing up a GitHub org --- README.markdown | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.markdown b/README.markdown index 9811cb8..0f57183 100644 --- a/README.markdown +++ b/README.markdown @@ -52,6 +52,23 @@ optional arguments: Then, put it in a cron job somewhere and forget about it for eternity. +How To Back Up Entire GitHub Organisation Repos +------------------------- + +1. Install Dependencies: `sudo pip install pygithub3] +2. Clone this repo using [git clone https://github.com/clockfort/GitHub-Backup.git +3. Just open the cloned repo folder and run the terminal: + +``` +./github-backup.py [Your GitHub Username] [Path To Saving Directory] -o [For Organisation] +``` + +Example: + +``` +./github-backup.py mohamed786 /home/mohamed786/githubbak -o LineageOS +``` + Why This Software Exists ------------------------- This software is useful in many cases: From 83174e1f6da022165f738ac350317a24efddd8dd Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 5 May 2019 02:07:11 +0200 Subject: [PATCH 12/18] improve style of README --- README.markdown | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.markdown b/README.markdown index 0f57183..6110be7 100644 --- a/README.markdown +++ b/README.markdown @@ -17,6 +17,7 @@ Installation is simple with Usage ---------------------------- + ```` usage: github-backup.py [-h] [-c] [-m] [-f] [-S] [-g ARGS] [-o ORG] [-s SUFFIX] [-P PREFIX] [-p PASSWORD] [-t TOKEN] username backupdir @@ -74,13 +75,9 @@ Why This Software Exists This software is useful in many cases: - GitHub suddenly explodes. - - GitHub goes out of business. - - Your corporation's backup policies are more stringent than GitHub's. - - You have spotty/no internet access - perhaps you'd like to have all of your repositories available to code on while you ride the train? - - You are paranoid tinfoil-hat wearer who needs to back up everything in triplicate on a variety of outdated tape media. From 577298c25b9d1202577d1fe03c4fc9c223be8279 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 5 May 2019 02:07:57 +0200 Subject: [PATCH 13/18] use requirements.txt --- README.markdown | 2 +- requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 requirements.txt diff --git a/README.markdown b/README.markdown index 6110be7..2b04d6a 100644 --- a/README.markdown +++ b/README.markdown @@ -13,7 +13,7 @@ GitHub-Backup requires `pygithub3` a Python library for the GitHub API v3. Installation is simple with - pip install pygithub3 + pip install -r requirements.txt Usage ---------------------------- diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ba82e82 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +pygithub3 From edf8db636beb3e3251f8922530c62fe66055471d Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 5 May 2019 03:22:37 +0200 Subject: [PATCH 14/18] use PyGitHub instead of pygithub3 --- github-backup.py | 123 ++++++++++++++++++++++++++--------------------- requirements.txt | 2 +- 2 files changed, 69 insertions(+), 56 deletions(-) diff --git a/github-backup.py b/github-backup.py index 3396a1f..d070389 100755 --- a/github-backup.py +++ b/github-backup.py @@ -8,104 +8,117 @@ Created: Fri Jun 15 2012 """ -from pygithub3 import Github +from github import Github from argparse import ArgumentParser import subprocess -import os +import os, os.path +import logging +LOGGER = logging.getLogger('github-backup') def main(): + logging.basicConfig(level=logging.INFO) + + parser = init_parser() args = parser.parse_args() - if not args.git: - args.git = [] + if args.quiet: + LOGGER.setLevel(logging.WARN) + elif args.debug: + LOGGER.setLevel(logging.DEBUG) # Process args - if args.cron: + if args.quiet: args.git.append("--quiet") - # Make the connection to Github here. - config = {'user': args.username} - args.backupdir = args.backupdir.rstrip("/") - if (args.token): - config['token'] = args.token + # Make the connection to Github here. + config = {'login_or_token': args.login_or_token} - if (args.password): + if args.password: config['password'] = args.password - config['login'] = args.username - - # if both password and token are specified, the token will be - # used, according to pygithub3 sources - # however, the username isn't required when using a token - if (args.token): - config['token'] = args.token gh = Github(**config) - # Get all repos - users = {} - user_repos = gh.repos.list(user=args.username).all() - for repo in user_repos: - if repo.owner.login not in users: - users[repo.owner.login] = gh.users.get(repo.owner.login) + # Check that backup dir exists + if not os.path.exists(args.backupdir): + os.mkdir(args.backupdir) - repo.user = users[repo.owner.login] - fullrepo = gh.repos.get(repo.owner.login, repo.name) - is_fork = hasattr(fullrepo, 'parent') and hasattr(fullrepo, 'source') - if not (args.skip_forks and is_fork): - process_repo(repo, args) + # Get all repos + filters = { + 'affiliation': ','.join(args.affiliation), + 'visibility': args.visibility + } + + if args.organization: + org = gh.get_org(args.org) + repos = org.get_repos(**filters) + else: + user = gh.get_user() + repos = user.get_repos(**filters) + + for repo in repos: + if args.skip_forks and repo.fork: + continue + + process_repo(repo, args) + + import sys + sys.exit(0) def init_parser(): - """ - set up the argument parser - """ + """Set up the argument parser.""" parser = ArgumentParser(description="makes a backup of all of a github user's repositories") - parser.add_argument("username", help="A Github username") + parser.add_argument("login_or_token", help="A Github username or token") parser.add_argument("backupdir", help="The folder where you want your backups to go") - parser.add_argument("-c", "--cron", help="Use this when running from a cron job", action="store_true") + parser.add_argument("-v", "--visibility", help="Filter repos by their visibility", choices=['all', 'public', 'private'], default='all') + parser.add_argument("-a", "--affiliation", help="Filter repos by their affiliation", action='append', type=str, default=['owner'], choices=['owner', 'collaborator', 'organization_member']) + parser.add_argument("-d", "--debug", help="Show debug info", action="store_true") + parser.add_argument("-q", "--quiet", help="Only show errors", action="store_true") parser.add_argument("-m", "--mirror", help="Create a bare mirror", action="store_true") parser.add_argument("-f", "--skip-forks", help="Skip forks", action="store_true") - parser.add_argument("-g", "--git", nargs="+", help="Pass extra arguments to git", default="", metavar="ARGS") - parser.add_argument("-S", "--ssh", help="Use SSH protocol", action="store_true") + parser.add_argument("-g", "--git", nargs="+", help="Pass extra arguments to git", type=list, default=[], metavar="ARGS") + parser.add_argument("-t", "--type", help="Select the protocol for cloning", choices=['git', 'http', 'ssh'], default='ssh') parser.add_argument("-s", "--suffix", help="Add suffix to repository directory names", default="") parser.add_argument("-p", "--password", help="Authenticate with Github API") parser.add_argument("-P", "--prefix", help="Add prefix to repository directory names", default="") - parser.add_argument("-t", "--token", help="Authenticate with Github API using OAuth token", default="") parser.add_argument("-o", "--organization", help="Backup Organizational repositories", metavar="ORG") return parser def process_repo(repo, args): - if not args.cron: - print("Processing repo: %s" % (repo.full_name)) + LOGGER.info("Processing repo: %s", repo.full_name) - dir = "%s/%s" % (args.backupdir, repo.name + args.suffix) + dir = args.backupdir + '/' + args.prefix + repo.name + args.suffix config = "%s/%s" % (dir, "config" if args.mirror else ".git/config") if not os.access(config, os.F_OK): - if not args.cron: - print("Repo doesn't exists, lets clone it") - + LOGGER.info("Repo doesn't exists, lets clone it") clone_repo(repo, dir, args) else: - if not args.cron: - print("Repo already exists, let's try to update it instead") - - update_repo(repo, dir, args) + LOGGER.info("Repo already exists, let's try to update it instead") + update_repo(repo, dir, args) def clone_repo(repo, dir, args): + if args.type == 'http': + url = repo.clone_url + elif args.type == 'ssh': + url = repo.ssh_url + elif args.type == 'git': + url = repo.git_url + + git_args = [url, os.path.basename(dir)] if args.mirror: - git("clone", ["--mirror", repo.git_url, dir], args.git, dir) - else: - git("clone", [repo.git_url, dir], args.git, dir) + git_args.insert(0, '--mirror') + + git("clone", git_args, args.git, args.backupdir) def update_repo(repo, dir, args): @@ -122,9 +135,9 @@ def update_repo(repo, dir, args): git("config", ["--local", "gitweb.description", repo.description.encode("utf-8")], gdir=dir) - if repo.user.name and repo.user.email: - owner = "%s <%s>" % (repo.user.name.encode("utf-8"), - repo.user.email.encode("utf-8")) + if repo.owner.name and repo.owner.email: + owner = "%s <%s>" % (repo.owner.name.encode("utf-8"), + repo.owner.email.encode("utf-8")) git("config", ["--local", "gitweb.owner", owner], gdir=dir) git("config", ["--local", "cgit.name", str(repo.name)], gdir=dir) @@ -135,12 +148,12 @@ def update_repo(repo, dir, args): def git(gcmd, args=[], gargs=[], gdir=""): cmd = ["git"] if gdir: - cmd.append("--git-dir") - cmd.append(gdir) + cmd.extend(["-C", gdir]) cmd.append(gcmd) cmd.extend(gargs) cmd.extend(args) + print(cmd) subprocess.call(cmd) if __name__ == "__main__": diff --git a/requirements.txt b/requirements.txt index ba82e82..65bf71b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -pygithub3 +PyGitHub From ac0f18d7b36548b50eee0502dd20ae3cf9b2ae59 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 5 May 2019 03:22:47 +0200 Subject: [PATCH 15/18] use PyGitHub instead of pygithub3 --- README.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index 2b04d6a..2713ba9 100644 --- a/README.markdown +++ b/README.markdown @@ -9,7 +9,7 @@ GitHub-Backup makes a local backup copy of all of a github user's (or github org Dependencies ---------------------------- -GitHub-Backup requires `pygithub3` a Python library for the GitHub API v3. +GitHub-Backup requires the `PyGitHub` Python package for the GitHub API v3. Installation is simple with From 16f237a1d24f2f451b26b05b72194ce46df0ad38 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 5 May 2019 03:22:58 +0200 Subject: [PATCH 16/18] update usage info in README.markdown --- README.markdown | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/README.markdown b/README.markdown index 2713ba9..a69de3c 100644 --- a/README.markdown +++ b/README.markdown @@ -19,34 +19,39 @@ Usage ---------------------------- ```` -usage: github-backup.py [-h] [-c] [-m] [-f] [-S] [-g ARGS] [-o ORG] [-s SUFFIX] [-P PREFIX] - [-p PASSWORD] [-t TOKEN] username backupdir +usage: github-backup.py [-h] [-v {all,public,private}] + [-a {owner,collaborator,organization_member}] [-d] + [-q] [-m] [-f] [-g ARGS [ARGS ...]] + [-t {git,http,ssh}] [-s SUFFIX] [-p PASSWORD] + [-P PREFIX] [-o ORG] + login_or_token backupdir makes a backup of all of a github user's repositories positional arguments: - username A Github username + login_or_token A Github username or token backupdir The folder where you want your backups to go optional arguments: - -h, --help Show this help message and exit - -c, --cron Use this when running from a cron job + -h, --help show this help message and exit + -v {all,public,private}, --visibility {all,public,private} + Filter repos by their visibility + -a {owner,collaborator,organization_member}, --affiliation {owner,collaborator,organization_member} + Filter repos by their affiliation + -d, --debug Show debug info + -q, --quiet Only show errors -m, --mirror Create a bare mirror -f, --skip-forks Skip forks - -S, --ssh Use SSH protocol - -g ARGS, --git ARGS Pass extra arguments to git - -o ORGANIZATION, --organization ORGANIZATION - Backup Organizational repositories - -p PASSWORD, --password PASSWORD - Authenticate with Github API - -P PREFIX, --prefix PREFIX - Add prefix to repository directory names + -g ARGS [ARGS ...], --git ARGS [ARGS ...] + Pass extra arguments to git + -t {git,http,ssh}, --type {git,http,ssh} + Select the protocol for cloning -s SUFFIX, --suffix SUFFIX Add suffix to repository directory names -p PASSWORD, --password PASSWORD - Authenticate with Github API using OAuth token - -t TOKEN, --token TOKEN - OAuth token for authentification + Authenticate with Github API + -P PREFIX, --prefix PREFIX + Add prefix to repository directory names -o ORG, --organization ORG Backup Organizational repositories ```` From 2b64e65b23ffa5f840fb6e680dc458fb063b8013 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 5 May 2019 03:23:53 +0200 Subject: [PATCH 17/18] rename README.markdown to README.md --- README.markdown => README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename README.markdown => README.md (100%) diff --git a/README.markdown b/README.md similarity index 100% rename from README.markdown rename to README.md From e833cf4815fbb93669132e0047e4c7246532ea45 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 5 May 2019 03:36:02 +0200 Subject: [PATCH 18/18] remove debugging helpers --- github-backup.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/github-backup.py b/github-backup.py index d070389..ff4c108 100755 --- a/github-backup.py +++ b/github-backup.py @@ -65,10 +65,6 @@ def main(): process_repo(repo, args) - import sys - sys.exit(0) - - def init_parser(): """Set up the argument parser."""