mirror of
https://github.com/clockfort/GitHub-Backup.git
synced 2025-03-16 00:00:06 +01:00
Merge pull request #30 from stv0g/rewrite
Rewrite: several new features
This commit is contained in:
Chris Lockfort
GitHub
commit
b899c46ff6
4 changed files with 221 additions and 149 deletions
|
|
@ -1,64 +0,0 @@
|
|||
GitHub-Backup
|
||||
=============
|
||||
|
||||
Idea/original implementation by Chris Lockfort (clockfort@csh.rit.edu) (Github username: Clockfort)
|
||||
|
||||
Python version by Anthony Gargiulo (anthony@agargiulo.com) (Github username: agargiulo)
|
||||
|
||||
Description
|
||||
----
|
||||
|
||||
GitHub-Backup makes a local backup copy of all of a github user's (or github organization's) repositories.
|
||||
|
||||
Dependencies
|
||||
----
|
||||
|
||||
GitHub-Backup requires `pygithub3` a Python library for the GitHub API v3.
|
||||
|
||||
Installation is simple with
|
||||
|
||||
pip install pygithub3
|
||||
|
||||
Usage
|
||||
-----
|
||||
````
|
||||
usage: github-backup.py [-h] [-c] [-m] [-g ARGS] [-s SUFFIX]
|
||||
username backupdir
|
||||
|
||||
makes a backup of all of a github user's repositories
|
||||
|
||||
positional arguments:
|
||||
username A Github username
|
||||
backupdir The folder where you want your backups to go
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
-c, --cron Use this when running from a cron job
|
||||
-m, --mirror Create a bare mirror
|
||||
-f, --skip-forks Skip forks
|
||||
-g ARGS, --git ARGS Pass extra arguments to git
|
||||
-s SUFFIX, --suffix SUFFIX
|
||||
Add suffix to repository directory names
|
||||
````
|
||||
|
||||
Then, put it in a cron job somewhere and forget about it for eternity.
|
||||
|
||||
Why This Software Exists
|
||||
-------------------------
|
||||
This software is useful in many cases:
|
||||
|
||||
- GitHub suddenly explodes.
|
||||
|
||||
- GitHub goes out of business.
|
||||
|
||||
- Your corporation's backup policies are more stringent than GitHub's.
|
||||
|
||||
- You have spotty/no internet access - perhaps you'd like to have all of your repositories available to code on while you ride the train?
|
||||
|
||||
- You are paranoid tinfoil-hat wearer who needs to back up everything in triplicate on a variety of outdated tape media.
|
||||
|
||||
|
||||
Questions, Improvements, Etc
|
||||
-----------------------------
|
||||
|
||||
If you have any improvements, I'm happy, (grateful, in fact) to entertain pull requests/patches, just drop me a line or message me on GitHub.
|
||||
106
README.md
Normal file
106
README.md
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
GitHub-Backup
|
||||
============================
|
||||
|
||||
Description
|
||||
----------------------------
|
||||
|
||||
GitHub-Backup makes a local backup copy of all of a github user's (or github organization's) repositories.
|
||||
|
||||
Dependencies
|
||||
----------------------------
|
||||
|
||||
GitHub-Backup requires the `PyGitHub` Python package for the GitHub API v3.
|
||||
|
||||
Installation is simple with
|
||||
|
||||
pip install -r requirements.txt
|
||||
|
||||
Usage
|
||||
----------------------------
|
||||
|
||||
````
|
||||
usage: github-backup.py [-h] [-v {all,public,private}]
|
||||
[-a {owner,collaborator,organization_member}] [-d]
|
||||
[-q] [-m] [-f] [-g ARGS [ARGS ...]]
|
||||
[-t {git,http,ssh}] [-s SUFFIX] [-p PASSWORD]
|
||||
[-P PREFIX] [-o ORG]
|
||||
login_or_token backupdir
|
||||
|
||||
makes a backup of all of a github user's repositories
|
||||
|
||||
positional arguments:
|
||||
login_or_token A Github username or token
|
||||
backupdir The folder where you want your backups to go
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
-v {all,public,private}, --visibility {all,public,private}
|
||||
Filter repos by their visibility
|
||||
-a {owner,collaborator,organization_member}, --affiliation {owner,collaborator,organization_member}
|
||||
Filter repos by their affiliation
|
||||
-d, --debug Show debug info
|
||||
-q, --quiet Only show errors
|
||||
-m, --mirror Create a bare mirror
|
||||
-f, --skip-forks Skip forks
|
||||
-g ARGS [ARGS ...], --git ARGS [ARGS ...]
|
||||
Pass extra arguments to git
|
||||
-t {git,http,ssh}, --type {git,http,ssh}
|
||||
Select the protocol for cloning
|
||||
-s SUFFIX, --suffix SUFFIX
|
||||
Add suffix to repository directory names
|
||||
-p PASSWORD, --password PASSWORD
|
||||
Authenticate with Github API
|
||||
-P PREFIX, --prefix PREFIX
|
||||
Add prefix to repository directory names
|
||||
-o ORG, --organization ORG
|
||||
Backup Organizational repositories
|
||||
````
|
||||
|
||||
Then, put it in a cron job somewhere and forget about it for eternity.
|
||||
|
||||
How To Back Up Entire GitHub Organisation Repos
|
||||
-------------------------
|
||||
|
||||
1. Install Dependencies: `sudo pip install pygithub3]
|
||||
2. Clone this repo using [git clone https://github.com/clockfort/GitHub-Backup.git
|
||||
3. Just open the cloned repo folder and run the terminal:
|
||||
|
||||
```
|
||||
./github-backup.py [Your GitHub Username] [Path To Saving Directory] -o [For Organisation]
|
||||
```
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
./github-backup.py mohamed786 /home/mohamed786/githubbak -o LineageOS
|
||||
```
|
||||
|
||||
Why This Software Exists
|
||||
-------------------------
|
||||
This software is useful in many cases:
|
||||
|
||||
- GitHub suddenly explodes.
|
||||
- GitHub goes out of business.
|
||||
- Your corporation's backup policies are more stringent than GitHub's.
|
||||
- You have spotty/no internet access - perhaps you'd like to have all of your repositories available to code on while you ride the train?
|
||||
- You are paranoid tinfoil-hat wearer who needs to back up everything in triplicate on a variety of outdated tape media.
|
||||
|
||||
|
||||
Questions, Improvements, Etc
|
||||
-----------------------------
|
||||
|
||||
If you have any improvements, I'm happy, (grateful, in fact) to entertain pull requests/patches, just drop me a line or message me on GitHub.
|
||||
|
||||
Contributors
|
||||
----------------------------
|
||||
|
||||
Idea/original implementation by
|
||||
|
||||
- Chris Lockfort (clockfort@csh.rit.edu) (Github: Clockfort)
|
||||
(Original idea)
|
||||
|
||||
- Anthony Gargiulo (anthony@agargiulo.com) (Github: agargiulo)
|
||||
(Python version)
|
||||
|
||||
- Steffen Vogel (post@steffenvogel.de) (Github: stv0g)
|
||||
(A lot of patches and improvements)
|
||||
199
github-backup.py
199
github-backup.py
|
|
@ -7,121 +7,150 @@ Authors: Anthony Gargiulo (anthony@agargiulo.com)
|
|||
Created: Fri Jun 15 2012
|
||||
"""
|
||||
|
||||
from pygithub3 import Github
|
||||
from argparse import ArgumentParser
|
||||
import os
|
||||
|
||||
from github import Github
|
||||
from argparse import ArgumentParser
|
||||
import subprocess
|
||||
import os, os.path
|
||||
import logging
|
||||
|
||||
LOGGER = logging.getLogger('github-backup')
|
||||
|
||||
def main():
|
||||
parser = init_parser()
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
# Process args
|
||||
if args.cron:
|
||||
args.git += "--quiet"
|
||||
|
||||
# Make the connection to Github here.
|
||||
config = { 'user': args.username }
|
||||
parser = init_parser()
|
||||
args = parser.parse_args()
|
||||
|
||||
if (args.password):
|
||||
config['password'] = args.password
|
||||
config['login'] = args.username
|
||||
if args.quiet:
|
||||
LOGGER.setLevel(logging.WARN)
|
||||
elif args.debug:
|
||||
LOGGER.setLevel(logging.DEBUG)
|
||||
|
||||
# if both password and token are specified, the token will be
|
||||
# used, according to pygithub3 sources
|
||||
# however, the username isn't required when using a token
|
||||
if (args.token):
|
||||
config['token'] = args.token
|
||||
# Process args
|
||||
if args.quiet:
|
||||
args.git.append("--quiet")
|
||||
|
||||
gh = Github(**config)
|
||||
args.backupdir = args.backupdir.rstrip("/")
|
||||
|
||||
# Get all of the given user's repos
|
||||
if args.organization:
|
||||
user_repos = gh.repos.list_by_org(args.organization).all()
|
||||
else:
|
||||
user_repos = gh.repos.list().all()
|
||||
# Make the connection to Github here.
|
||||
config = {'login_or_token': args.login_or_token}
|
||||
|
||||
for repo in user_repos:
|
||||
repo.user = gh.users.get(repo.owner.login)
|
||||
fullrepo = gh.repos.get(repo.owner.login, repo.name)
|
||||
if not (args.skip_forks and hasattr(fullrepo, 'parent') and hasattr(fullrepo, 'source')):
|
||||
process_repo(repo, args)
|
||||
if args.password:
|
||||
config['password'] = args.password
|
||||
|
||||
gh = Github(**config)
|
||||
|
||||
# Check that backup dir exists
|
||||
if not os.path.exists(args.backupdir):
|
||||
os.mkdir(args.backupdir)
|
||||
|
||||
# Get all repos
|
||||
filters = {
|
||||
'affiliation': ','.join(args.affiliation),
|
||||
'visibility': args.visibility
|
||||
}
|
||||
|
||||
if args.organization:
|
||||
org = gh.get_org(args.org)
|
||||
repos = org.get_repos(**filters)
|
||||
else:
|
||||
user = gh.get_user()
|
||||
repos = user.get_repos(**filters)
|
||||
|
||||
for repo in repos:
|
||||
if args.skip_forks and repo.fork:
|
||||
continue
|
||||
|
||||
process_repo(repo, args)
|
||||
|
||||
def init_parser():
|
||||
"""
|
||||
set up the argument parser
|
||||
"""
|
||||
"""Set up the argument parser."""
|
||||
|
||||
parser = ArgumentParser(description="makes a backup of all of a github user's repositories")
|
||||
parser = ArgumentParser(description="makes a backup of all of a github user's repositories")
|
||||
|
||||
parser.add_argument("username", help="A Github username")
|
||||
parser.add_argument("backupdir", help="The folder where you want your backups to go")
|
||||
parser.add_argument("-c","--cron", help="Use this when running from a cron job", action="store_true")
|
||||
parser.add_argument("-m","--mirror", help="Create a bare mirror", action="store_true")
|
||||
parser.add_argument("-f","--skip-forks", help="Skip forks", action="store_true")
|
||||
parser.add_argument("-g","--git", help="Pass extra arguments to git", default="", metavar="ARGS")
|
||||
parser.add_argument("-s", "--suffix", help="Add suffix to repository directory names", default="")
|
||||
parser.add_argument("-p", "--password", help="Authenticate with Github API")
|
||||
parser.add_argument("-P","--prefix", help="Add prefix to repository directory names", default="")
|
||||
parser.add_argument("-o","--organization", help="Backup Organizational repositories")
|
||||
parser.add_argument("-S","--ssh", help="Use SSH protocol", action="store_true")
|
||||
parser.add_argument("-t","--token", help="Authenticate with Github API using OAuth token", default="")
|
||||
parser.add_argument("login_or_token", help="A Github username or token")
|
||||
parser.add_argument("backupdir", help="The folder where you want your backups to go")
|
||||
parser.add_argument("-v", "--visibility", help="Filter repos by their visibility", choices=['all', 'public', 'private'], default='all')
|
||||
parser.add_argument("-a", "--affiliation", help="Filter repos by their affiliation", action='append', type=str, default=['owner'], choices=['owner', 'collaborator', 'organization_member'])
|
||||
parser.add_argument("-d", "--debug", help="Show debug info", action="store_true")
|
||||
parser.add_argument("-q", "--quiet", help="Only show errors", action="store_true")
|
||||
parser.add_argument("-m", "--mirror", help="Create a bare mirror", action="store_true")
|
||||
parser.add_argument("-f", "--skip-forks", help="Skip forks", action="store_true")
|
||||
parser.add_argument("-g", "--git", nargs="+", help="Pass extra arguments to git", type=list, default=[], metavar="ARGS")
|
||||
parser.add_argument("-t", "--type", help="Select the protocol for cloning", choices=['git', 'http', 'ssh'], default='ssh')
|
||||
parser.add_argument("-s", "--suffix", help="Add suffix to repository directory names", default="")
|
||||
parser.add_argument("-p", "--password", help="Authenticate with Github API")
|
||||
parser.add_argument("-P", "--prefix", help="Add prefix to repository directory names", default="")
|
||||
parser.add_argument("-o", "--organization", help="Backup Organizational repositories", metavar="ORG")
|
||||
|
||||
return parser
|
||||
|
||||
return parser
|
||||
|
||||
def process_repo(repo, args):
|
||||
if not args.cron:
|
||||
print("Processing repo: %s"%(repo.full_name))
|
||||
LOGGER.info("Processing repo: %s", repo.full_name)
|
||||
|
||||
dir = "%s/%s"%(args.backupdir, args.prefix + repo.name + args.suffix)
|
||||
config = "%s/%s"%(dir, "config" if args.mirror else ".git/config")
|
||||
dir = args.backupdir + '/' + args.prefix + repo.name + args.suffix
|
||||
config = "%s/%s" % (dir, "config" if args.mirror else ".git/config")
|
||||
|
||||
if not os.access(config, os.F_OK):
|
||||
if not args.cron: print("Repo doesn't exists, lets clone it")
|
||||
clone_repo(repo, dir, args)
|
||||
else:
|
||||
if not args.cron: print("Repo already exists, let's try to update it instead")
|
||||
|
||||
update_repo(repo, dir, args)
|
||||
if not os.access(config, os.F_OK):
|
||||
LOGGER.info("Repo doesn't exists, lets clone it")
|
||||
clone_repo(repo, dir, args)
|
||||
else:
|
||||
LOGGER.info("Repo already exists, let's try to update it instead")
|
||||
update_repo(repo, dir, args)
|
||||
|
||||
|
||||
def clone_repo(repo, dir, args):
|
||||
if args.mirror:
|
||||
options = args.git + " --mirror"
|
||||
else:
|
||||
options = args.git
|
||||
if args.type == 'http':
|
||||
url = repo.clone_url
|
||||
elif args.type == 'ssh':
|
||||
url = repo.ssh_url
|
||||
elif args.type == 'git':
|
||||
url = repo.git_url
|
||||
|
||||
os.system('git clone %s %s %s'%(options, repo.ssh_url if args.ssh else repo.git_url, dir))
|
||||
git_args = [url, os.path.basename(dir)]
|
||||
if args.mirror:
|
||||
git_args.insert(0, '--mirror')
|
||||
|
||||
git("clone", git_args, args.git, args.backupdir)
|
||||
|
||||
|
||||
def update_repo(repo, dir, args):
|
||||
savedPath = os.getcwd()
|
||||
os.chdir(dir)
|
||||
# GitHub => Local
|
||||
# TODO: use subprocess package and fork git into
|
||||
# background (major performance boost expected)
|
||||
if args.mirror:
|
||||
git("fetch", ["--prune"], args.git, dir)
|
||||
else:
|
||||
git("pull", gargs=args.git, gdir=dir)
|
||||
|
||||
# GitHub => Local
|
||||
# TODO: use subprocess package and fork git into background (major performance boost expected)
|
||||
if args.mirror:
|
||||
os.system("git fetch %s"%(args.git + " --prune",))
|
||||
else:
|
||||
os.system("git pull %s"%(args.git,))
|
||||
# Fetch description and owner (useful for gitweb, cgit etc.)
|
||||
if repo.description:
|
||||
git("config", ["--local", "gitweb.description",
|
||||
repo.description.encode("utf-8")], gdir=dir)
|
||||
|
||||
# Fetch description and owner (useful for gitweb, cgit etc.)
|
||||
# TODO: can we combine that in a single call to 'git config'
|
||||
if repo.description is not None:
|
||||
os.system("git config --local gitweb.description %s"%(shell_escape(repo.description),))
|
||||
if repo.user.name is not None and repo.user.email is not None:
|
||||
os.system("git config --local gitweb.owner %s"%(shell_escape("%s <%s>"%(repo.user.name, repo.user.email.encode("utf-8"))),))
|
||||
if repo.owner.name and repo.owner.email:
|
||||
owner = "%s <%s>" % (repo.owner.name.encode("utf-8"),
|
||||
repo.owner.email.encode("utf-8"))
|
||||
git("config", ["--local", "gitweb.owner", owner], gdir=dir)
|
||||
|
||||
os.system("git config --local cgit.name %s"%(shell_escape(repo.name),))
|
||||
os.system("git config --local cgit.defbranch %s"%(shell_escape(repo.default_branch),))
|
||||
os.system("git config --local cgit.clone-url %s"%(shell_escape(repo.clone_url),))
|
||||
|
||||
os.chdir(savedPath)
|
||||
git("config", ["--local", "cgit.name", str(repo.name)], gdir=dir)
|
||||
git("config", ["--local", "cgit.defbranch", str(repo.default_branch)], gdir=dir)
|
||||
git("config", ["--local", "cgit.clone-url", str(repo.clone_url)], gdir=dir)
|
||||
|
||||
def shell_escape(str):
|
||||
if str:
|
||||
return "'" + unicode(str.replace("'", "'\\''")).encode("utf-8") + "'"
|
||||
|
||||
def git(gcmd, args=[], gargs=[], gdir=""):
|
||||
cmd = ["git"]
|
||||
if gdir:
|
||||
cmd.extend(["-C", gdir])
|
||||
cmd.append(gcmd)
|
||||
cmd.extend(gargs)
|
||||
cmd.extend(args)
|
||||
|
||||
print(cmd)
|
||||
subprocess.call(cmd)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
main()
|
||||
|
|
|
|||
1
requirements.txt
Normal file
1
requirements.txt
Normal file
|
|
@ -0,0 +1 @@
|
|||
PyGitHub
|
||||
Loading…
Add table
Reference in a new issue