Add remirror

2 files changed, 429 insertions, 0 deletions

diff --git a/remirror/remirror b/remirror/remirror
new file mode 100755
index 0000000..8ca3062
--- /dev/null
+++ b/remirror/remirror
@@ -0,0 +1,369 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# (c) 2017, Vlad Glagolev <stealth@sourcemage.org>
+
+import argparse
+import os
+import shlex
+import stat
+import subprocess
+import sys
+
+from multiprocessing import Process, Queue, cpu_count
+
+try:
+    import yaml
+
+    HAS_YAML = True
+except ImportError:
+    HAS_YAML = False
+
+try:
+    import json as js
+
+    HAS_JSON = True
+except ImportError:
+    try:
+        import simplejson as js
+    except ImportError:
+        HAS_JSON = False
+
+try:
+    import requests
+
+    HAS_REQUESTS = True
+except ImportError:
+    HAS_REQUESTS = False
+
+
+__version__ = "0.0.1"  # major.minor.revision
+
+
+# ~/.sourcemage/mirror.yaml
+_DEFAULT_CONFIG = os.path.join(os.path.expanduser("~"), ".sourcemage/mirror.yaml")
+
+_CONFIG_FORMAT = {
+    'root': '',
+    'projects': ('name', 'repos'),
+    'mirrors': ('name', 'username', 'token')
+}
+
+
+class ConfigError(Exception):
+    """ Configuration file error class """
+
+    def __init__(self, message):
+        sys.stderr.write("%s: configuration error: %s\n" % (os.path.basename(sys.argv[0]), message))
+
+        exit(2)
+
+
+def check_mode(f):
+    st = os.stat(f)
+
+    if st.st_mode & (stat.S_IRGRP | stat.S_IROTH):
+        ConfigError("configuration file is group/world-readable")
+
+
+def configure(fp):
+    """ Check configuration file syntax. """
+
+    check_mode(fp.name)
+
+    try:
+        config = yaml.safe_load(fp)
+    except yaml.YAMLError as e:
+        ConfigError("syntax mismatch in configuration file '%s': %s" % (fp.name, e))
+
+    # should be replaced with schema (https://github.com/keleshev/schema)
+    for section in _CONFIG_FORMAT:
+        if section not in config:
+            ConfigError("missing section: %s" % section)
+
+        if section in ('projects', 'mirrors'):
+            if type(config[section]) != list:
+                ConfigError("section '%s' must be a list")
+
+            for key in _CONFIG_FORMAT[section]:
+                for item in config[section]:
+                    if key not in item:
+                        ConfigError("missing key in section's '%s' item: %s" % (section, key))
+
+                    if key == 'repos':
+                        if type(item[key]) != list:
+                            ConfigError("value for '%s' must be a list" % key)
+
+                        for repo in item[key]:
+                            if type(repo) != dict:
+                                ConfigError("repository item must be a dictionary")
+
+                            for k, v in repo.items():
+                                if (type(k) and type(v)) != str:
+                                    ConfigError("key/value types must be strings")
+                    else:
+                        if type(item[key]) != str:
+                            ConfigError("value for '%s' must be a string" % key)
+
+    return config
+
+
+class ReMirror(Process):
+
+    def __init__(self, queue):
+        Process.__init__(self)
+
+        self.queue = queue
+
+    def run(self):
+        while True:
+            repo = self.queue.get()
+
+            if repo is None:
+                break
+
+            repo.sync()
+
+        return
+
+
+class Repo(object):
+
+    def __init__(self, repo, project, conf):
+        self.name = repo[0]
+        self.fullpath = os.path.join(conf['root'], repo[1] + '.git')
+        self.project = project
+        self.conf = conf
+
+    def sync(self):
+        for p_conf in self.conf['mirrors']:
+            p_name = p_conf['name'].lower()
+
+            if p_name == "github":
+                provider = ProviderGitHub(p_conf)
+            elif p_name == "bitbucket":
+                provider = ProviderBitbucket(p_conf)
+            else:
+                provider = Provider(p_conf)
+
+                provider.log("unsupported mirror provider detected: '%s'" % p_name)
+
+                continue
+
+            provider.mirror(self)
+
+
+class Provider(object):
+    api_url = None
+    git_host = None
+
+    def __init__(self, conf):
+        self.conf = conf
+        self.auth = None
+        self.group = None
+
+    def mirror(self, repo):
+        if not self.group:
+            self.error("owner org/group setting for repository '%s' not found, skipping" % repo.name)
+
+        try:
+            self.sanity_check(repo)
+        except Exception as e:
+            self.error("unable to mirror repository '%s': %s" % (repo.name, e))
+
+        self.push(repo)
+
+    def sanity_check(self, repo):
+        self.error("sanity check not implemented")
+
+    def http_call(self, url, payload=None):
+        if self.api_url is None:
+            raise Exception("HTTP API calls are not implemented")
+
+        req_url = self.api_url + url.lstrip('/')
+
+        # POST
+        if payload:
+            req = requests.post(req_url, json=payload, **self.auth)
+
+            self.log("performed %s request: %s" % (req.request.method, req_url))
+        # GET
+        else:
+            req = requests.get(req_url, **self.auth)
+
+        return req
+
+    def push(self, repo):
+        git_repo = "%s:%s/%s.git" % (self.git_host, self.group, repo.name)
+        git_cmd = "git push --mirror %s" % git_repo
+
+        sub_cmd = shlex.split(git_cmd)
+
+        try:
+            self.log("pushing to %s\n" % git_repo)
+
+            sub = subprocess.Popen(sub_cmd, stdout=subprocess.PIPE,
+                                   stderr=subprocess.PIPE, cwd=repo.fullpath)
+        except (OSError, IOError) as e:
+            self.error(e)
+
+        stdout, stderr = sub.communicate()
+
+        if sub.returncode != 0:
+            self.error("pushing to %s failed: %s" % (git_repo, stderr))
+        else:
+            if stderr != '':
+                self.log(stderr)
+
+            self.log("successful push to %s\n--" % git_repo)
+
+    def log(self, message):
+        sys.stdout.write("%s\n" % message)
+
+    def error(self, message):
+        sys.stderr.write("Error: %s\n" % message)
+
+        return
+
+
+class ProviderGitHub(Provider):
+    api_url = "https://api.github.com/"
+    git_host = "git@github.com"
+
+    def __init__(self, conf):
+        super(ProviderGitHub, self).__init__(conf)
+
+        self.auth = {'headers': {"Authorization": "token %s" % self.conf['token']}}
+        self.group = self.conf.get('organization')
+        self.team_id = self.conf.get('team_id')
+
+    def sanity_check(self, repo):
+        repo_ok = "/repos/{0}/{1}".format(self.group, repo.name)
+
+        req = self.http_call(repo_ok)
+
+        if req.status_code == 404:
+            repo_mk = "/orgs/{0}/repos".format(self.group)
+
+            payload = {'name': repo.name,
+                       'has_wiki': False,
+                       'has_issues': False}
+
+            if self.team_id:
+                payload['team_id'] = self.team_id
+
+            req = self.http_call(repo_mk, payload)
+
+            req.raise_for_status()
+        else:
+            req.raise_for_status()
+
+
+class ProviderBitbucket(Provider):
+    api_url = "https://api.bitbucket.org/2.0/"
+    git_host = "git@bitbucket.org"
+
+    def __init__(self, conf):
+        super(ProviderBitbucket, self).__init__(conf)
+
+        self.auth = {'auth': (self.conf['username'], self.conf['token'])}
+        self.group = self.conf.get('team')
+
+    def sanity_check(self, repo):
+        repo_ok = "/repositories/{0}/{1}".format(self.group, repo.name)
+
+        req = self.http_call(repo_ok)
+
+        if req.status_code == 404:
+            payload = {'scm': "git",
+                       'has_wiki': False,
+                       'has_issues': False}
+
+            if repo.project[1] is not None:
+                project_ok = "/teams/{0}/projects/{1}".format(self.group, repo.project[1])
+
+                req = self.http_call(project_ok)
+
+                if req.status_code == 404:
+                    project_mk = project_ok[:project_ok.rfind('/') + 1]
+
+                    payload_proj = {'name': repo.project[0],
+                                    'key': repo.project[1]}
+
+                    req = self.http_call(project_mk, payload_proj)
+
+                    req.raise_for_status()
+                else:
+                    req.raise_for_status()
+
+                payload['project'] = {'key': repo.project[1]}
+
+            repo_mk = repo_ok
+
+            req = self.http_call(repo_mk, payload)
+
+            req.raise_for_status()
+        else:
+            req.raise_for_status()
+
+
+def mirror(conf):
+    queue = Queue()
+
+    proc_num = cpu_count() * 2
+
+    repos = []
+
+    for p in conf['projects']:
+        project = p['name'], p.get('key')
+
+        for r in p['repos']:
+            repos.append(Repo(r.popitem(), project, conf))
+
+    procs = []
+
+    for _ in xrange(proc_num):
+        proc = ReMirror(queue)
+
+        procs.append(proc)
+
+        proc.start()
+
+    for repo in repos:
+        queue.put(repo)
+
+    # poison pill
+    for _ in xrange(proc_num):
+        queue.put(None)
+
+    for proc in procs:
+        proc.join()
+
+
+def main():
+    if not HAS_YAML:
+        ConfigError("pyyaml (http://pyyaml.org/) is mandatory for the run")
+
+    if not HAS_JSON:
+        ConfigError("built-in json or external simplejson module is missing")
+
+    if not HAS_REQUESTS:
+        ConfigError("requests (http://python-requests.org/) is mandatory for the run")
+
+    parser = argparse.ArgumentParser(description='Repository mirror tool')
+
+    parser.add_argument("-v", "--version", action='version',
+                        version='%(prog)s {0}'.format(__version__))
+    parser.add_argument("-c", "--config", type=argparse.FileType('r'),
+                        default=_DEFAULT_CONFIG,
+                        help="configuration file in YAML format (default: %(default)s)")
+
+    args = parser.parse_args()
+
+    config = configure(args.config)
+
+    mirror(config)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/remirror/remirror.config.yaml b/remirror/remirror.config.yaml
new file mode 100644
index 0000000..21edb07
--- /dev/null
+++ b/remirror/remirror.config.yaml
@@ -0,0 +1,60 @@
+# Configuration file for Source Mage GNU/Linux repository mirroring;
+# should be copied to ~/.sourcemage/mirror.yaml
+---
+root: /srv/git/smgl
+
+projects:
+  - name: Codex
+    key: SMCDX
+    repos:
+      - grimoire: grimoire
+      - grimoire-xorg-modular: grimoire/xorg-modular
+      - grimoire-z-rejected: grimoire/z-rejected
+      - grimoire-games: grimoire/games
+      - grimoire-binary: grimoire/binary
+      - grimoire-p4_history: grimoire/grimoire-p4_history
+
+  - name: Cauldron
+    key: SMCLD
+    repos:
+      - cauldron: cauldron
+
+  - name: Tome
+    key: SMTM
+    repos:
+      - tome-rdp: tome/rdp
+      - tome-scrolls: tome/scrolls
+
+  - name: Sorcery
+    key: SMSRC
+    repos:
+      - sorcery: sorcery
+
+  - name: Wand
+    key: SMWND
+    repos:
+      - wand: wand
+
+  - name: Miscellaneous
+    key: SMMSC
+    repos:
+      - archspecs: misc/archspecs
+      - bashdoc: misc/bashdoc
+      - castfs: misc/castfs
+      - enthrall: misc/enthrall
+      - guru-tools: misc/guru-tools
+      - licenses: misc/licenses
+      - quill: misc/quill
+      - prometheus: misc/prometheus
+
+mirrors:
+  - name: github
+    username: magesync
+    token: secret
+    organization: sourcemage
+    team_id: 0
+
+  - name: bitbucket
+    username: magesync
+    token: secret
+    team: sourcemage