diff --git a/propagator/remotes/github.py b/propagator/remotes/github.py index 0b87b30..af88562 100644 --- a/propagator/remotes/github.py +++ b/propagator/remotes/github.py @@ -1,128 +1,128 @@ # This file is part of Propagator, a KDE Sysadmin Project # # Copyright (C) 2015-2016 Boudhayan Gupta # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # 3. Neither the name of KDE e.V. (or its successor approved by the # membership of KDE e.V.) nor the names of its contributors may be used # to endorse or promote products derived from this software without # specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import re import os import sys import requests try: import simplejson as json except ImportError: import json from propagator.core.sync import restricted_sync from propagator.core.config import config_general from propagator.remotes.remotebase import RemoteBase class Remote(RemoteBase): ENDPOINT_REPO = "https://api.github.com/repos" ENDPOINT_ORGS = "https://api.github.com/orgs" @property def plugin_name(self): return "github" def plugin_init(self, *args, **kwargs): cfgpath = os.path.expanduser("~/.propagator/remotes_github.json") with open(cfgpath) as f: cfgdict = json.load(f) self.access_token = cfgdict["access_token"] self.organization = cfgdict["organization"] self.except_checks = ( re.compile(i) for i in cfgdict["excepts"] ) self.repo_base = config_general.get("repobase") self.session = requests.Session() self.session.headers.update({"Accept": "application/vnd.github.v3+json"}) self.session.headers.update({"Authorization": " ".join(("token", self.access_token))}) def _strip_reponame(self, name): if name.endswith(".git"): return name[:-4] return name def _repo_exists(self, name): url = "{0}/{1}/{2}".format(self.ENDPOINT_REPO, self.organization, name) r = self.session.get(url) return ((r.ok) and ("id" in r.json().keys())) def can_handle_repo(self, name): name = self._strip_reponame(name) for i in self.except_checks: if (i.match(name)): return False return True def create(self, name, desc = "This repository has no description"): name = self._strip_reponame(name) # if the repo already exists, exit early if self._repo_exists(name): return True # build up the create request and execute it payload = { "name": name, "description": desc, "private": False, "has_issues": False, "has_wiki": False, "has_downloads": False, "auto_init": False, } url = "{0}/{1}/{2}".format(self.ENDPOINT_ORGS, self.organization, "repos") r = self.session.post(url, data = json.dumps(payload)) return ((r.status_code == 201) and ("id" in r.json().keys())) def rename(self, name, dest): name = self._strip_reponame(name) dest = self._strip_reponame(dest) payload = { "name": dest } url = "{0}/{1}/{2}".format(self.ENDPOINT_REPO, self.organization, name) r = self.session.patch(url, data = json.dumps(payload)) return ((r.status_code == 201) and ("id" in r.json().keys())) def update(self, repo, name): srcdir = os.path.join(self.repo_base, name) desturl = "git@github.com:{0}/{1}".format(self.organization, name) if not self._repo_exists(name): self.create(name, repo.description) return restricted_sync(srcdir, desturl) def delete(self, name): name = self._strip_reponame(name) url = "{0}/{1}/{2}".format(self.ENDPOINT_REPO, self.organization, name) r = self.session.delete(url) return (r.status_code == 204) def setdesc(self, name, desc): name = self._strip_reponame(name) - payload = { "description": desc } + payload = { "name": name, "description": desc } url = "{0}/{1}/{2}".format(self.ENDPOINT_REPO, self.organization, name) r = self.session.patch(url, data = json.dumps(payload)) return ((r.status_code == 201) and ("id" in r.json().keys())) diff --git a/propagator/utils/common.py b/propagator/utils/common.py index 61a753d..85a981b 100644 --- a/propagator/utils/common.py +++ b/propagator/utils/common.py @@ -1,41 +1,71 @@ # This file is part of Propagator, a KDE Sysadmin Project # # Copyright 2015 Boudhayan Gupta # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # 3. Neither the name of KDE e.V. (or its successor approved by the # membership of KDE e.V.) nor the names of its contributors may be used # to endorse or promote products derived from this software without # specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import git import os +try: + import simplejson as json +except ImportError: + import json + from propagator.core.config import config_general +from propagator.remoteslave import amqp + +# Data First + +allowed_ops = ("create", "update", "rename", "delete", "syncdesc") + +# Methods + +def send_message(payload): + body = json.dumps(payload) + channel = amqp.create_channel_producer() + return channel.basic_publish( + exchange = amqp.exchange_name(), + routing_key = "", + body = body + ) def is_valid_repo(repo): path = os.path.join(config_general.get("repobase"), repo) try: repo = git.Repo(path) except (git.exc.NoSuchPathError, git.exc.InvalidGitRepositoryError): return False return True + +def set_local_desc(repo, desc): + path = os.path.join(config_general.get("repobase"), repo) + repo = git.Repo(path) + try: + repo.description = desc + except Exception: + return False + return True diff --git a/propagator/utils/mirrorsync.py b/propagator/utils/mirrorctl.py similarity index 58% copy from propagator/utils/mirrorsync.py copy to propagator/utils/mirrorctl.py index b46ff20..af2c4fd 100644 --- a/propagator/utils/mirrorsync.py +++ b/propagator/utils/mirrorctl.py @@ -1,72 +1,77 @@ # This file is part of Propagator, a KDE Sysadmin Project # -# Copyright 2015 Boudhayan Gupta +# Copyright 2016 Boudhayan Gupta # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # 3. Neither the name of KDE e.V. (or its successor approved by the # membership of KDE e.V.) nor the names of its contributors may be used # to endorse or promote products derived from this software without # specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import sys import argparse -try: - import simplejson as json -except ImportError: - import json - -from propagator.utils.common import is_valid_repo -from propagator.remoteslave import amqp +from propagator.utils.common import is_valid_repo, send_message, allowed_ops, set_local_desc def cmdline_process(): - parser = argparse.ArgumentParser(description = "Sync updates to all repository mirrors through Propagator") + parser = argparse.ArgumentParser(description = "Propagator Mirror Control Utility") + parser.add_argument("action", type = str, help = "the action to perform on the repo", choices = allowed_ops) parser.add_argument("reponame", type = str, help = "the name of the repository to update") + parser.add_argument("-D", "--dest", type = str, help = "the new name for the renamed repository", required = False) + parser.add_argument("-d", "--desc", type = str, help = "the new description for the repository", required = False) parser.add_argument("remote", type = str, nargs = "*", help = "update only these remotes") - parser.add_argument("-v", "--verbose", action = "store_true", help = "give verbose output on the standard output") + args = parser.parse_args() + if (args.action == "rename") and (not args.dest): + parser.error("The rename action requires a specified restination") return args -def send_message(payload): - body = json.dumps(payload) - channel = amqp.create_channel_producer() - return channel.basic_publish( - exchange = amqp.exchange_name(), - routing_key = "", - body = body - ) - def main(): args = cmdline_process() - if not (is_valid_repo(args.reponame)): + if (args.action == "update") and (not is_valid_repo(args.reponame)): print("ERROR: {} is not a valid repository".format(args.reponame), file = sys.stderr) sys.exit(1) - message = { "operation": "update", "repository": args.reponame, "attempt": 0 } + + # compose the message + message = { + "operation": args.action, + "repository": args.reponame, + "attempt": 0 + } + + # compose optional extras in the message + if (args.action == "rename"): + message["destination"] = args.dest + if (args.action == "syncdesc") and args.desc: + if not set_local_desc(args.reponame, args.desc): + print("ERROR: Unable to set description on the local repository") + sys.exit(1) if args.remote: - message["remote_for"] = parser.remote + message["remote_for"] = args.remote + + # send the message and analyse the fallout ret = send_message(message) if not ret: - print("ERROR: Failed to notify Propagator to update repository mirrors") + print("ERROR: Failed to notify Propagator. Please check if the AMQP server is running") sys.exit(1) - if args.verbose: - print("Successfully notified Propagator to update repository mirrors") + print("SUCCESS: Propagator has been notified") sys.exit(0) diff --git a/propagator/utils/mirrorsync.py b/propagator/utils/mirrorsync.py index b46ff20..6c8b531 100644 --- a/propagator/utils/mirrorsync.py +++ b/propagator/utils/mirrorsync.py @@ -1,72 +1,57 @@ # This file is part of Propagator, a KDE Sysadmin Project # -# Copyright 2015 Boudhayan Gupta +# Copyright 2016 Boudhayan Gupta # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # 3. Neither the name of KDE e.V. (or its successor approved by the # membership of KDE e.V.) nor the names of its contributors may be used # to endorse or promote products derived from this software without # specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import sys import argparse -try: - import simplejson as json -except ImportError: - import json - -from propagator.utils.common import is_valid_repo -from propagator.remoteslave import amqp +from propagator.utils.common import is_valid_repo, send_message def cmdline_process(): parser = argparse.ArgumentParser(description = "Sync updates to all repository mirrors through Propagator") parser.add_argument("reponame", type = str, help = "the name of the repository to update") parser.add_argument("remote", type = str, nargs = "*", help = "update only these remotes") parser.add_argument("-v", "--verbose", action = "store_true", help = "give verbose output on the standard output") args = parser.parse_args() return args -def send_message(payload): - body = json.dumps(payload) - channel = amqp.create_channel_producer() - return channel.basic_publish( - exchange = amqp.exchange_name(), - routing_key = "", - body = body - ) - def main(): args = cmdline_process() if not (is_valid_repo(args.reponame)): print("ERROR: {} is not a valid repository".format(args.reponame), file = sys.stderr) sys.exit(1) message = { "operation": "update", "repository": args.reponame, "attempt": 0 } if args.remote: - message["remote_for"] = parser.remote + message["remote_for"] = args.remote ret = send_message(message) if not ret: print("ERROR: Failed to notify Propagator to update repository mirrors") sys.exit(1) if args.verbose: print("Successfully notified Propagator to update repository mirrors") sys.exit(0) diff --git a/setup.py b/setup.py index 9d166db..495e8c4 100644 --- a/setup.py +++ b/setup.py @@ -1,33 +1,34 @@ from setuptools import setup, find_packages setup( name = "propagator", - version = "0.1.95", + version = "0.1.96", author = "Boudhayan Gupta", author_email = "bgupta@kde.org", description = ("A git mirror fleet manager"), license = "BSD", keywords = "git mirror devops", url = "http://www.kde.org/", packages = find_packages(), install_requires = ( "simplejson", "GitPython", "pika", "logbook", "circus" ), classifiers = ( "Development Status :: 4 - Beta", "License :: OSI Approved :: BSD License", "Topic :: Software Development :: Version Control", "Topic :: System :: Software Distribution" ), entry_points = { "console_scripts": ( "propagator-agent = propagator.agent:main", "propagator-remoteslave = propagator.remoteslave:main", - "propagator-mirrorsync = propagator.utils.mirrorsync:main" + "propagator-mirrorsync = propagator.utils.mirrorsync:main", + "propagator-mirrorctl = propagator.utils.mirrorctl:main" ), }, )