diff --git a/README.markdown b/README.markdown deleted file mode 100644 index daff30a..0000000 --- a/README.markdown +++ /dev/null @@ -1,64 +0,0 @@ -GitHub-Backup -============= - -Idea/original implementation by Chris Lockfort (clockfort@csh.rit.edu) (Github username: Clockfort) - -Python version by Anthony Gargiulo (anthony@agargiulo.com) (Github username: agargiulo) - -Description ----- - -GitHub-Backup makes a local backup copy of all of a github user's (or github organization's) repositories. - -Dependencies ----- - -GitHub-Backup requires `pygithub3` a Python library for the GitHub API v3. - -Installation is simple with - - pip install pygithub3 - -Usage ------ -```` -usage: github-backup.py [-h] [-c] [-m] [-g ARGS] [-s SUFFIX] - username backupdir - -makes a backup of all of a github user's repositories - -positional arguments: - username A Github username - backupdir The folder where you want your backups to go - -optional arguments: - -h, --help show this help message and exit - -c, --cron Use this when running from a cron job - -m, --mirror Create a bare mirror - -f, --skip-forks Skip forks - -g ARGS, --git ARGS Pass extra arguments to git - -s SUFFIX, --suffix SUFFIX - Add suffix to repository directory names -```` - -Then, put it in a cron job somewhere and forget about it for eternity. - -Why This Software Exists -------------------------- -This software is useful in many cases: - - - GitHub suddenly explodes. - - - GitHub goes out of business. - - - Your corporation's backup policies are more stringent than GitHub's. - - - You have spotty/no internet access - perhaps you'd like to have all of your repositories available to code on while you ride the train? - - - You are paranoid tinfoil-hat wearer who needs to back up everything in triplicate on a variety of outdated tape media. - - -Questions, Improvements, Etc ------------------------------ - -If you have any improvements, I'm happy, (grateful, in fact) to entertain pull requests/patches, just drop me a line or message me on GitHub. diff --git a/README.md b/README.md new file mode 100644 index 0000000..a69de3c --- /dev/null +++ b/README.md @@ -0,0 +1,106 @@ +GitHub-Backup +============================ + +Description +---------------------------- + +GitHub-Backup makes a local backup copy of all of a github user's (or github organization's) repositories. + +Dependencies +---------------------------- + +GitHub-Backup requires the `PyGitHub` Python package for the GitHub API v3. + +Installation is simple with + + pip install -r requirements.txt + +Usage +---------------------------- + +```` +usage: github-backup.py [-h] [-v {all,public,private}] + [-a {owner,collaborator,organization_member}] [-d] + [-q] [-m] [-f] [-g ARGS [ARGS ...]] + [-t {git,http,ssh}] [-s SUFFIX] [-p PASSWORD] + [-P PREFIX] [-o ORG] + login_or_token backupdir + +makes a backup of all of a github user's repositories + +positional arguments: + login_or_token A Github username or token + backupdir The folder where you want your backups to go + +optional arguments: + -h, --help show this help message and exit + -v {all,public,private}, --visibility {all,public,private} + Filter repos by their visibility + -a {owner,collaborator,organization_member}, --affiliation {owner,collaborator,organization_member} + Filter repos by their affiliation + -d, --debug Show debug info + -q, --quiet Only show errors + -m, --mirror Create a bare mirror + -f, --skip-forks Skip forks + -g ARGS [ARGS ...], --git ARGS [ARGS ...] + Pass extra arguments to git + -t {git,http,ssh}, --type {git,http,ssh} + Select the protocol for cloning + -s SUFFIX, --suffix SUFFIX + Add suffix to repository directory names + -p PASSWORD, --password PASSWORD + Authenticate with Github API + -P PREFIX, --prefix PREFIX + Add prefix to repository directory names + -o ORG, --organization ORG + Backup Organizational repositories +```` + +Then, put it in a cron job somewhere and forget about it for eternity. + +How To Back Up Entire GitHub Organisation Repos +------------------------- + +1. Install Dependencies: `sudo pip install pygithub3] +2. Clone this repo using [git clone https://github.com/clockfort/GitHub-Backup.git +3. Just open the cloned repo folder and run the terminal: + +``` +./github-backup.py [Your GitHub Username] [Path To Saving Directory] -o [For Organisation] +``` + +Example: + +``` +./github-backup.py mohamed786 /home/mohamed786/githubbak -o LineageOS +``` + +Why This Software Exists +------------------------- +This software is useful in many cases: + + - GitHub suddenly explodes. + - GitHub goes out of business. + - Your corporation's backup policies are more stringent than GitHub's. + - You have spotty/no internet access - perhaps you'd like to have all of your repositories available to code on while you ride the train? + - You are paranoid tinfoil-hat wearer who needs to back up everything in triplicate on a variety of outdated tape media. + + +Questions, Improvements, Etc +----------------------------- + +If you have any improvements, I'm happy, (grateful, in fact) to entertain pull requests/patches, just drop me a line or message me on GitHub. + +Contributors +---------------------------- + +Idea/original implementation by + +- Chris Lockfort (clockfort@csh.rit.edu) (Github: Clockfort) + (Original idea) + +- Anthony Gargiulo (anthony@agargiulo.com) (Github: agargiulo) + (Python version) + +- Steffen Vogel (post@steffenvogel.de) (Github: stv0g) + (A lot of patches and improvements) diff --git a/github-backup.py b/github-backup.py index 5e5923b..ff4c108 100755 --- a/github-backup.py +++ b/github-backup.py @@ -7,121 +7,150 @@ Authors: Anthony Gargiulo (anthony@agargiulo.com) Created: Fri Jun 15 2012 """ -from pygithub3 import Github -from argparse import ArgumentParser -import os +from github import Github +from argparse import ArgumentParser +import subprocess +import os, os.path +import logging + +LOGGER = logging.getLogger('github-backup') def main(): - parser = init_parser() - args = parser.parse_args() + logging.basicConfig(level=logging.INFO) - # Process args - if args.cron: - args.git += "--quiet" - # Make the connection to Github here. - config = { 'user': args.username } + parser = init_parser() + args = parser.parse_args() - if (args.password): - config['password'] = args.password - config['login'] = args.username + if args.quiet: + LOGGER.setLevel(logging.WARN) + elif args.debug: + LOGGER.setLevel(logging.DEBUG) - # if both password and token are specified, the token will be - # used, according to pygithub3 sources - # however, the username isn't required when using a token - if (args.token): - config['token'] = args.token + # Process args + if args.quiet: + args.git.append("--quiet") - gh = Github(**config) + args.backupdir = args.backupdir.rstrip("/") - # Get all of the given user's repos - if args.organization: - user_repos = gh.repos.list_by_org(args.organization).all() - else: - user_repos = gh.repos.list().all() + # Make the connection to Github here. + config = {'login_or_token': args.login_or_token} - for repo in user_repos: - repo.user = gh.users.get(repo.owner.login) - fullrepo = gh.repos.get(repo.owner.login, repo.name) - if not (args.skip_forks and hasattr(fullrepo, 'parent') and hasattr(fullrepo, 'source')): - process_repo(repo, args) + if args.password: + config['password'] = args.password + gh = Github(**config) + + # Check that backup dir exists + if not os.path.exists(args.backupdir): + os.mkdir(args.backupdir) + + # Get all repos + filters = { + 'affiliation': ','.join(args.affiliation), + 'visibility': args.visibility + } + + if args.organization: + org = gh.get_org(args.org) + repos = org.get_repos(**filters) + else: + user = gh.get_user() + repos = user.get_repos(**filters) + + for repo in repos: + if args.skip_forks and repo.fork: + continue + + process_repo(repo, args) def init_parser(): - """ - set up the argument parser - """ + """Set up the argument parser.""" - parser = ArgumentParser(description="makes a backup of all of a github user's repositories") + parser = ArgumentParser(description="makes a backup of all of a github user's repositories") - parser.add_argument("username", help="A Github username") - parser.add_argument("backupdir", help="The folder where you want your backups to go") - parser.add_argument("-c","--cron", help="Use this when running from a cron job", action="store_true") - parser.add_argument("-m","--mirror", help="Create a bare mirror", action="store_true") - parser.add_argument("-f","--skip-forks", help="Skip forks", action="store_true") - parser.add_argument("-g","--git", help="Pass extra arguments to git", default="", metavar="ARGS") - parser.add_argument("-s", "--suffix", help="Add suffix to repository directory names", default="") - parser.add_argument("-p", "--password", help="Authenticate with Github API") - parser.add_argument("-P","--prefix", help="Add prefix to repository directory names", default="") - parser.add_argument("-o","--organization", help="Backup Organizational repositories") - parser.add_argument("-S","--ssh", help="Use SSH protocol", action="store_true") - parser.add_argument("-t","--token", help="Authenticate with Github API using OAuth token", default="") + parser.add_argument("login_or_token", help="A Github username or token") + parser.add_argument("backupdir", help="The folder where you want your backups to go") + parser.add_argument("-v", "--visibility", help="Filter repos by their visibility", choices=['all', 'public', 'private'], default='all') + parser.add_argument("-a", "--affiliation", help="Filter repos by their affiliation", action='append', type=str, default=['owner'], choices=['owner', 'collaborator', 'organization_member']) + parser.add_argument("-d", "--debug", help="Show debug info", action="store_true") + parser.add_argument("-q", "--quiet", help="Only show errors", action="store_true") + parser.add_argument("-m", "--mirror", help="Create a bare mirror", action="store_true") + parser.add_argument("-f", "--skip-forks", help="Skip forks", action="store_true") + parser.add_argument("-g", "--git", nargs="+", help="Pass extra arguments to git", type=list, default=[], metavar="ARGS") + parser.add_argument("-t", "--type", help="Select the protocol for cloning", choices=['git', 'http', 'ssh'], default='ssh') + parser.add_argument("-s", "--suffix", help="Add suffix to repository directory names", default="") + parser.add_argument("-p", "--password", help="Authenticate with Github API") + parser.add_argument("-P", "--prefix", help="Add prefix to repository directory names", default="") + parser.add_argument("-o", "--organization", help="Backup Organizational repositories", metavar="ORG") + + return parser - return parser def process_repo(repo, args): - if not args.cron: - print("Processing repo: %s"%(repo.full_name)) + LOGGER.info("Processing repo: %s", repo.full_name) - dir = "%s/%s"%(args.backupdir, args.prefix + repo.name + args.suffix) - config = "%s/%s"%(dir, "config" if args.mirror else ".git/config") + dir = args.backupdir + '/' + args.prefix + repo.name + args.suffix + config = "%s/%s" % (dir, "config" if args.mirror else ".git/config") - if not os.access(config, os.F_OK): - if not args.cron: print("Repo doesn't exists, lets clone it") - clone_repo(repo, dir, args) - else: - if not args.cron: print("Repo already exists, let's try to update it instead") - - update_repo(repo, dir, args) + if not os.access(config, os.F_OK): + LOGGER.info("Repo doesn't exists, lets clone it") + clone_repo(repo, dir, args) + else: + LOGGER.info("Repo already exists, let's try to update it instead") + update_repo(repo, dir, args) def clone_repo(repo, dir, args): - if args.mirror: - options = args.git + " --mirror" - else: - options = args.git + if args.type == 'http': + url = repo.clone_url + elif args.type == 'ssh': + url = repo.ssh_url + elif args.type == 'git': + url = repo.git_url - os.system('git clone %s %s %s'%(options, repo.ssh_url if args.ssh else repo.git_url, dir)) + git_args = [url, os.path.basename(dir)] + if args.mirror: + git_args.insert(0, '--mirror') + + git("clone", git_args, args.git, args.backupdir) def update_repo(repo, dir, args): - savedPath = os.getcwd() - os.chdir(dir) + # GitHub => Local + # TODO: use subprocess package and fork git into + # background (major performance boost expected) + if args.mirror: + git("fetch", ["--prune"], args.git, dir) + else: + git("pull", gargs=args.git, gdir=dir) - # GitHub => Local - # TODO: use subprocess package and fork git into background (major performance boost expected) - if args.mirror: - os.system("git fetch %s"%(args.git + " --prune",)) - else: - os.system("git pull %s"%(args.git,)) + # Fetch description and owner (useful for gitweb, cgit etc.) + if repo.description: + git("config", ["--local", "gitweb.description", + repo.description.encode("utf-8")], gdir=dir) - # Fetch description and owner (useful for gitweb, cgit etc.) - # TODO: can we combine that in a single call to 'git config' - if repo.description is not None: - os.system("git config --local gitweb.description %s"%(shell_escape(repo.description),)) - if repo.user.name is not None and repo.user.email is not None: - os.system("git config --local gitweb.owner %s"%(shell_escape("%s <%s>"%(repo.user.name, repo.user.email.encode("utf-8"))),)) + if repo.owner.name and repo.owner.email: + owner = "%s <%s>" % (repo.owner.name.encode("utf-8"), + repo.owner.email.encode("utf-8")) + git("config", ["--local", "gitweb.owner", owner], gdir=dir) - os.system("git config --local cgit.name %s"%(shell_escape(repo.name),)) - os.system("git config --local cgit.defbranch %s"%(shell_escape(repo.default_branch),)) - os.system("git config --local cgit.clone-url %s"%(shell_escape(repo.clone_url),)) - - os.chdir(savedPath) + git("config", ["--local", "cgit.name", str(repo.name)], gdir=dir) + git("config", ["--local", "cgit.defbranch", str(repo.default_branch)], gdir=dir) + git("config", ["--local", "cgit.clone-url", str(repo.clone_url)], gdir=dir) -def shell_escape(str): - if str: - return "'" + unicode(str.replace("'", "'\\''")).encode("utf-8") + "'" + +def git(gcmd, args=[], gargs=[], gdir=""): + cmd = ["git"] + if gdir: + cmd.extend(["-C", gdir]) + cmd.append(gcmd) + cmd.extend(gargs) + cmd.extend(args) + + print(cmd) + subprocess.call(cmd) if __name__ == "__main__": - main() + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..65bf71b --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +PyGitHub