diff --git a/.gitignore b/.gitignore index c60af00..6b4a261 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,93 @@ -py3env/* -*.pyc -supervisord.pid + +# Created by https://www.gitignore.io/api/python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# IPython Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# dotenv +.env + +# virtualenv +venv/ +ENV/ + +# Spyder project settings +.spyderproject + +# Rope project settings +.ropeproject diff --git a/README.md b/README.md deleted file mode 100644 index 53f40f2..0000000 --- a/README.md +++ /dev/null @@ -1,58 +0,0 @@ -![Propagator - A KDE Project](docs/logo.png "Propagator - A KDE Project") - -## What is Propagator? - -Propagator is a Git mirror fleet manager. Every time someone pushes a commit to a central read-write Git server, Propagator propagates that commit to a fleet of read-only git mirror servers, as well as to GitHub. - -Propagator was built with ♥ at [KDE](https://www.kde.org/), where we use it to propagate updates in near real-time to our fleet of `anongit.kde.org` servers, as well as to our read-only [GitHub mirror](https://github.com/kde/). - -## Quickstart - -Propagator runs on your main Git server, i.e., the one that has the repositories on disk, and to which users can push commits. This is important, since Propagator is notified of repository updates through the repo's `post-recieve` or `post-update` hooks. - -Running Propagator is easy. You'll need to have to following installed: - -* Python 3.5 (Propagator uses the AsyncIO module) -* RabbitMQ -* Celery 3 -* GitPython -* OpenSSH Client -* Supervisord - -To install, simply clone this repository locally, `cd` into it, and edit the configuration at `config/ServerConfig.json`. Then you can run the server by executing: - - $: supervisord -c ./supervisord.conf - -The default configuration shipped in the repository explicitly disables daemonization of the Supervisord process, but you can re-enable that in `supervisord.conf`. - -## Anongit Servers - -On the Anongit servers (mirrors), we use a small SSH agent to remotely create, move, or delete repositories, or change descriptions. - -The agent is available at `agent/GatorSSHAgent`, and is a single self-contained Python 3.3+ script whose only external dependency is GitPython. You can simply copy this script to your Anongit servers independently of the rest of the project. - -To set up the agent to talk to the server, here's what you have to do: - -1. First, generate a new SSH keypair and copy the public key to the server's `~/.ssh/authorized_keys` file. Do **NOT** re-use your existing SSH key. This is important. -2. On the Propagator server, edit the configuration to point to the private key file. The relevant entry is `AnongitAPIKeyFile` in `config/ServerConfig.json`. -3. On the mirror, edit the `~/.ssh/authorized_keys` and **prefix** the entry for this public key with `command=/path/to/GatorSSHAgent`. The entry should look like this: `command=/path/to/GatorSSHAgent ssh-rsa AbbcDEfgggHHj... your@ssh-key-comment`. - -Of course, this only handles the management of repositories on the servers. Pushes are done over the standard ssh+git mechanism. This is why you need a separate key for the control API - OpenSSH uses the key to discern between a shell or git login or a control API login. - -## Updating Repositories - -Once the server is up and running, and Anongit servers have been configured with the SSH agent, you'll want to actually push updates to your fleet. - -Propagator can create repositories on remotes on first push, so you don't have to create repositories on the remotes manually. - -In your repo's `post-recieve` or `post-update` hook, simply add this command: - - ${PROPAGATOR_REPO}/bin/mirrorctl update reponame.git - -Where `${PROPAGATOR_REPO}` is the location to where you've cloned the Propagator repository, and `reponame.git` is the repository that you want to update on your remotes. - -And that's it. - -## Maintainership - -Propagator is currently maintained by Boudhayan Gupta (). It is not part of any KDE release module, and tarballs are released independently as and when major releases are warranted. diff --git a/agent/GatorSSHAgent b/agent/GatorSSHAgent deleted file mode 100755 index a48e780..0000000 --- a/agent/GatorSSHAgent +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/python3 -# This file is part of Propagator, a KDE Sysadmin Project -# -# Copyright 2015 Boudhayan Gupta -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# 3. Neither the name of KDE e.V. (or its successor approved by the -# membership of KDE e.V.) nor the names of its contributors may be used -# to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR -# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import os -import sys -import shlex -import shutil -import git - -# the GATOR_REPOS_ROOT environment variable should tell us where the repos are -# located. if the variable is not defined, default to ~/repos - -REPO_ROOT = os.environ.get("GATOR_REPOS_ROOT") -if not REPO_ROOT: - REPO_ROOT = "~/repos" -REPO_ROOT = os.path.expanduser(REPO_ROOT) - -# write a description to the repo's description file. repos on the server are -# always bare repos, so this file exists - -def setRepoDescription(repoName, repoDesc): - - repoDir = os.path.join(REPO_ROOT, repoName) - if not os.path.exists(repoDir): - print("ERROR: Repo does not exist") - return False - - with open(os.path.join(repoDir, "description"), "w") as f: - f.write(repoDesc.strip() + "\n") - return True - -# create a bare repo on the server - -def createRepo(repoName, repoDesc): - - repoDir = os.path.join(REPO_ROOT, repoName) - if os.path.exists(repoDir): - print("ERROR: Repo already exists") - return False - - repo = git.Repo.init(repoDir, bare = True) - if repo.bare: - return setRepoDescription(repoName, repoDesc) - print("ERROR: Failed to create repo") - return False - -# boolean function to check if a repo already exists. this is slightly tailored -# for the EXISTS command, so we can't use this function internally to do exists -# checks in the other functions - -def repoExists(repoName): - - repoDir = os.path.join(REPO_ROOT, repoName) - if os.path.exists(repoDir): - try: - repo = git.Repo(repoDir) - except git.exc.InvalidGitRepositoryError: - print("ERROR: Repo does not exist") - return False - return True - - print("ERROR: Repo does not exist") - return False - -# move or rename an existing repo - -def moveRepo(oldRepo, newRepo): - - oldRepoPath = os.path.abspath(os.path.join(REPO_ROOT, oldRepo)) - newRepoPath = os.path.abspath(os.path.join(REPO_ROOT, newRepo)) - - if not os.path.exists(oldRepoPath): - print("ERROR: Source repo does not exist") - return False - - if os.path.exists(newRepoPath): - print("ERROR: Destination repo already exists") - return False - - basePath = os.path.dirname(newRepoPath) - if not os.path.exists(basePath): - os.makedirs(basePath) - shutil.move(oldRepoPath, newRepoPath) - return True - -# delete an existing repo. this does not ask for confirmation, so be careful - -def deleteRepo(repoName): - - repoDir = os.path.join(REPO_ROOT, repoName) - if not os.path.exists(repoDir): - print("ERROR: Repo does not exist") - return False - - shutil.rmtree(repoDir) - return not os.path.exists(repoDir) - -# the SSH_ORIGINAL_COMMAND parser - -def processCommand(commandList): - - cmd = commandList[0] - repoName = commandList[1] - - if cmd == "EXISTS": - return repoExists(repoName) - elif cmd == "SETDESC": - repoDesc = " ".join(commandList[2:]) - return setRepoDescription(repoName, repoDesc) - elif cmd == "CREATE": - repoDesc = " ".join(commandList[2:]) - return createRepo(repoName, repoDesc) - elif cmd == "DELETE": - return deleteRepo(repoName) - elif cmd == "MOVE": - newRepo = commandList[2] - return moveRepo(repoName, newRepo) - else: - print("ERROR: Invalid command entered. This account does not allow shell access.") - return False - -# SSH_ORIGINAL_COMMAND contains our command string. parse it and pass it to -# processCommand to actually do something - -if __name__ == "__main__": - - soc = os.environ.get("SSH_ORIGINAL_COMMAND") - if not soc: - print("ERROR: Invalid command entered. This account does not allow shell access.") - print("FAIL") - sys.exit(1) - - socList = shlex.split(soc) - ret = processCommand(socList) - if ret: - print("OK") - sys.exit(0) - print("FAIL") - sys.exit(1) diff --git a/bin/mirrorctl b/bin/mirrorctl deleted file mode 100755 index 2a1f054..0000000 --- a/bin/mirrorctl +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/python3 -# This file is part of Propagator, a KDE Sysadmin Project -# -# Copyright (C) 2015-2016 Boudhayan Gupta -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# 3. Neither the name of KDE e.V. (or its successor approved by the -# membership of KDE e.V.) nor the names of its contributors may be used -# to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR -# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from datetime import datetime - -import socket -import os -import sys -import argparse - -# propagator daemon server details, and client log path - -SERVER_ADDR = "::1" -SERVER_PORT = 58192 -CLIENT_LOGF = os.environ.get("GATOR_MIRRORCTL_LOG", os.path.expanduser("~/.propagator/mirrorctl.log")) - -# utility functions - -def LogAction(message, iserror, verbose): - - # if verbose mode is enabled and not an error, print to stdout - if (not iserror) and (verbose): - print(message) - - # if error, print to stderr - if (iserror): - print(message, file = sys.stderr) - - # always log to the logfile. create path first if not exists - logpath = os.path.dirname(CLIENT_LOGF) - if not os.path.isdir(logpath): - os.makedirs(logpath, 0o755, True) - - # and write the logline - with open(CLIENT_LOGF, "a") as f: - logline = "{0} | {1}\n".format(datetime.now().strftime("%Y-%m-%d %k:%M:%S"), message) - f.write(logline) - -def SendCommand(cmd, verbose = True): - - # initialise socket to connect to propagator daemon - clientSocket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) - clientSocket.settimeout(5) - - # try to connect to propagator daemon - try: - clientSocket.connect((SERVER_ADDR, SERVER_PORT)) - except Exception as exc: - LogAction("FATAL: Failed to connect to Propagator Daemon. The error was: {0}".format(str(exc)), True, verbose) - sys.exit(1) - - # send our computed command - try: - clientSocket.sendall(cmd.encode()) - except Exception as exc: - LogAction("FATAL: Could not send command to Propagator Daemon. The error was: {0}".format(str(exc)), True, verbose) - finally: - clientSocket.close() - LogAction("Command sent to Propagator Daemon: {0}".format(cmd.strip()), False, verbose) - -def ComputeCommand(results): - - cmd = results.command.lower() - if cmd == "create": - return "create {0}\r\n".format(results.name) - elif cmd == "update": - return "update {0}\r\n".format(results.name) - elif cmd == "create": - return "delete {0}\r\n".format(results.name) - elif cmd == "move": - return "move {0} {1}\r\n".format(results.name, results.destination) - else: - return None - -def SanityCheck(repo): - - # TODO: basic sanity checking - check that the repo exists - return True - -def ParseArguments(): - - # create our main parser - parser = argparse.ArgumentParser(prog = "mirrorctl") - parser.add_argument("-q", "--quiet", action = "store_true", help = "don't display any output when no errors are encountered") - subparsers = parser.add_subparsers(dest = "command") - - # subparser for repo creation - pCreate = subparsers.add_parser("create", help = "create repos on mirrors for already existing repo on the master server") - pCreate.add_argument("name", type = str, help = "the name of the repo to create, without path, with trailing .git") - - # subparser for repo move - pmove = subparsers.add_parser("move", help = "move a repo on all mirrors") - pmove.add_argument("name", type = str, help = "the source repo, without path, with trailing .git") - pmove.add_argument("destination", type = str, help = "the destination repo, without path, with trailing .git") - - # subparser for repo update - pUpdate = subparsers.add_parser("update", help = "push updates to all repo mirrors") - pUpdate.add_argument("name", type = str, help = "the repo to update, without path, with trailing .git") - - # subparser for repo delete - pDelete = subparsers.add_parser("delete", help = "delete a repo on all mirrors") - pDelete.add_argument("name", type = str, help = "the repo to delete, without path, with trailing .git") - - # parse all arguments - results = parser.parse_args() - - # if invalid or no subcommand was supplied, the invocation is invalid - if not results.command or results.command.lower() not in ("create", "move", "update", "delete"): - parser.print_help() - sys.exit(1) - - # return the parse results - return results - -if __name__ == "__main__": - - # run the argument parser - results = ParseArguments() - verbose = not results.quiet - - # check the the repo exists and is valid - if not SanityCheck(results.name): - LogAction("FATAL: \"{0}\" either does not exist or is not a valid repository".format(results.name), True, verbose) - sys.exit(1) - - # compute the command and send it - command = ComputeCommand(results) - SendCommand(command, verbose) - - # done - sys.exit(0) diff --git a/config/MasterConfig.json b/config/MasterConfig.json deleted file mode 100644 index 4d40948..0000000 --- a/config/MasterConfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "ServerListenIP": "::1", - "ServerListenPort": 58192, - "ControlListenIP": "::1", - "ControlListenPort": 58193, - "RedisQueueKey": "GatorQueue", - "RepoRoot": "/home/bg14ina/testRepos/" -} diff --git a/config/RemotesGithub.json b/config/RemotesGithub.json deleted file mode 100644 index 545fb4d..0000000 --- a/config/RemotesGithub.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "organization": "BaloneyGeekCorp", - "accesstoken": "dummytoken", - - "excepts": [ - "^gitolite-admin.git$", - "([a-zA-Z0-9]*)/(.*)" - ] -} diff --git a/docs/logo.png b/docs/logo.png deleted file mode 100644 index 3fb2699..0000000 Binary files a/docs/logo.png and /dev/null differ diff --git a/logs/.gitignore b/logs/.gitignore deleted file mode 100644 index d6b7ef3..0000000 --- a/logs/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore diff --git a/propagator/__init__.py b/propagator/__init__.py new file mode 100644 index 0000000..f62c007 --- /dev/null +++ b/propagator/__init__.py @@ -0,0 +1 @@ +__all__ = ("agent",) diff --git a/server/GatorServer.py b/propagator/agent/__init__.py similarity index 64% rename from server/GatorServer.py rename to propagator/agent/__init__.py index 7604fff..5fd6517 100644 --- a/server/GatorServer.py +++ b/propagator/agent/__init__.py @@ -1,44 +1,56 @@ -#!/usr/bin/python3 -# This file is part of Propagator, a KDE Sysadmin Project +# This file is part of Propagator, a KDE project # -# Copyright 2015-2016 (C) Boudhayan Gupta +# Copyright 2015 Boudhayan Gupta # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # 3. Neither the name of KDE e.V. (or its successor approved by the # membership of KDE e.V.) nor the names of its contributors may be used # to endorse or promote products derived from this software without # specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import os import sys -import tornado.ioloop +import shlex -# set up logging -from logbook import StreamHandler -StreamHandler(sys.stdout).push_application() +from . import config +from . import repo +from . import gitcmd +from . import control -# start the command server -import CommandServer -cmdServer = CommandServer.CommandServer() -cmdServer.listen(58192, "::1") +def main(): + cmd = os.environ.get("SSH_ORIGINAL_COMMAND") + if not cmd: + print("Sorry, this account does not provide shell access.") + sys.exit(1) -# start the ioloop -tornado.ioloop.IOLoop.current().start() + entry = shlex.split(cmd)[0] + if entry == "anongitctl": + if not control.handle_command(cmd): + print("FAIL") + sys.exit(192) + print("OK") + sys.exit(0) + elif entry in ("git-receive-pack", "git-upload-pack", "git-upload-archive"): + gitcmd.handle_command(cmd) + sys.exit(192) + else: + print("ERROR: This command cannot be accepted. This account does not provide shell access", file = sys.stderr) diff --git a/server/CommandServer.py b/propagator/agent/config.py similarity index 66% rename from server/CommandServer.py rename to propagator/agent/config.py index 92b4574..46ba417 100644 --- a/server/CommandServer.py +++ b/propagator/agent/config.py @@ -1,48 +1,52 @@ -# This file is part of Propagator, a KDE Sysadmin Project +# This file is part of Propagator, a KDE project # -# Copyright (C) 2015-2016 Boudhayan Gupta +# Copyright 2015 Boudhayan Gupta # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # 3. Neither the name of KDE e.V. (or its successor approved by the # membership of KDE e.V.) nor the names of its contributors may be used # to endorse or promote products derived from this software without # specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import sys +import configparser +import os -import tornado.tcpserver -import tornado.gen +def repobase(): + default = os.path.expanduser("~/repositories") + cfgpath = os.path.expanduser("~/.propagator/anongit.cfg") + try: + config = configparser.ConfigParser() + config.read(cfgpath) + path = config["anongit"].get("repobase", default) + return os.path.expanduser(path) + except: + return default -from CommandProtocol import ParseCommand, ExecuteCommand -from CommandProtocol import PropagatorProtocolException - -class CommandServer(tornado.tcpserver.TCPServer): - - @tornado.gen.coroutine - def handle_stream(self, stream, address): - data = yield stream.read_until_close() - try: - context = ParseCommand(data.decode().strip()) - except PropagatorProtocolException as exc: - sys.stderr.write(exc.logline()) - return - ExecuteCommand(context) +def translate_path(path): + path = os.path.normpath(path) + if path.startswith(".."): + head, tail = os.path.split(path) + if not tail: + return None + path = tail + joined = os.path.join(repobase(), path) + return os.path.normpath(joined) diff --git a/propagator/agent/control.py b/propagator/agent/control.py new file mode 100644 index 0000000..5121188 --- /dev/null +++ b/propagator/agent/control.py @@ -0,0 +1,126 @@ +# This file is part of Propagator, a KDE project +# +# Copyright 2015 Boudhayan Gupta +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of KDE e.V. (or its successor approved by the +# membership of KDE e.V.) nor the names of its contributors may be used +# to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR +# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import shlex +import shutil + +from . import repo +from . import config + +def help(): + print("Propagator - A git mirror fleet manager") + print("anongitctl help") + print("") + print("Subcommands:") + print(" create [description] - create repository with description") + print(" rename - move/rename and existing repository") + print(" delete - delete a repository") + print(" setdesc - set the description of an existing repository") + print("") + +def setdesc(path, desc): + if not repo.set_repo_description(path, desc): + return False + print("OK: Description successfully set.") + return True + +def create(path, desc = None): + if not repo.create(path, False): + return False + print("OK: Repository successfully created.") + if desc: + return setdesc(path, desc) + return True + +def rename(oldpath, newpath): + if not os.path.exists(oldpath): + print("ERROR: Source repo does not exist.") + return False + if os.path.exists(newpath): + print("ERROR: Destination repo already exists.") + return False + basepath = os.path.dirname(newpath) + if not os.path.exists(basepath): + os.makedirs(basepath) + shutil.move(oldpath, newpath) + print("OK: Repository successfully renamed.") + return True + +def delete(path): + if not os.path.exists(path): + print("ERROR: Repository does not exist.") + return False + shutil.rmtree(path) + print("OK: Repository successfully deleted.") + return True + +def handle_command(cmd): + cmd_parts = shlex.split(cmd) + if not cmd_parts[1] in ("create", "rename", "delete", "setdesc", "help"): + print("ERROR: Invalid anongitctl subcommand.") + help() + return False + if cmd_parts[1] == "help": + help() + return True + + try: + reponame = config.translate_path(cmd_parts[2]) + except IndexError: + print("ERROR: Incorrect command syntax.") + help() + return False + + if cmd_parts[1] == "create": + try: + desc = cmd_parts[3] + except IndexError: + desc = None + return create(reponame, desc) + elif cmd_parts[1] == "rename": + try: + newname = config.translate_path(cmd_parts[3]) + except IndexError: + print("ERROR: Incorrect command syntax.") + help() + return False + return rename(reponame, newname) + elif cmd_parts[1] == "delete": + return delete(reponame) + elif cmd_parts[1] == "setdesc": + try: + desc = cmd_parts[3] + except IndexError: + print("ERROR: Incorrect command syntax.") + help() + return False + return setdesc(reponame, desc) + print("ERROR: Unknown error.") + return False diff --git a/server/SyncJob.py b/propagator/agent/gitcmd.py similarity index 60% copy from server/SyncJob.py copy to propagator/agent/gitcmd.py index 5e6620b..b97f6c1 100644 --- a/server/SyncJob.py +++ b/propagator/agent/gitcmd.py @@ -1,64 +1,68 @@ -# This file is part of Propagator, a KDE Sysadmin Project +# This file is part of Propagator, a KDE project # # Copyright 2015 Boudhayan Gupta # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # 3. Neither the name of KDE e.V. (or its successor approved by the # membership of KDE e.V.) nor the names of its contributors may be used # to endorse or promote products derived from this software without # specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import git +import shlex +import os +import sys -def doSync(src, dest, restricted = False): +from . import config +from . import repo - # init the git repo and remote objects first +def analyse_command(cmd): + cmd_parts_old = shlex.split(cmd) + cmd_parts_new = [] + repopath = None - repo = git.Repo(src) - remote = git.Remote(repo, dest) + for item in cmd_parts_old: + if item.endswith(".git"): + repopath = config.translate_path(item) + cmd_parts_new.append("'{}'".format(repopath)) + else: + cmd_parts_new.append(item) + if not repopath: + return False - # if we're doing a restricted push, we only update branches and tags. - # everything else needs to be ignored + cmdstring = " ".join(cmd_parts_new) + return (repopath, cmdstring) - refs = [] +def handle_command(cmd): + try: + repopath, cmdstring = analyse_command(cmd) + except TypeError: + return False + repopath = config.translate_path(repopath) - if (restricted): - ret = remote.push(("--mirror", "--dry-run")) - for info in ret: - # check if the local ref is either a head or a tag - if type(info.local_ref) in (git.refs.Head, git.refs.TagReference): - refs.append("".join(("+", info.local_ref.name, ":", info.remote_ref_string))) - - # do the push - - ret = None - if (refs): - ret = remote.push(refs) - else: - ret = remote.push("--mirror") - - # analyse the results and return success or failure - - for info in st: - if (info.flags & git.PushInfo.ERROR): + if cmdstring.startswith("git-receive-pack"): + ret = repo.create(repopath) + if not ret: + print("ERROR: The remote repository does not exist and could not be created", file = sys.stderr) return False - return True + + args = ["git-shell", "-c", cmdstring] + os.execvp("git-shell", args) diff --git a/server/SyncJob.py b/propagator/agent/repo.py similarity index 61% rename from server/SyncJob.py rename to propagator/agent/repo.py index 5e6620b..1557851 100644 --- a/server/SyncJob.py +++ b/propagator/agent/repo.py @@ -1,64 +1,59 @@ -# This file is part of Propagator, a KDE Sysadmin Project +# This file is part of Propagator, a KDE project # # Copyright 2015 Boudhayan Gupta # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # 3. Neither the name of KDE e.V. (or its successor approved by the # membership of KDE e.V.) nor the names of its contributors may be used # to endorse or promote products derived from this software without # specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import git - -def doSync(src, dest, restricted = False): - - # init the git repo and remote objects first - - repo = git.Repo(src) - remote = git.Remote(repo, dest) - - # if we're doing a restricted push, we only update branches and tags. - # everything else needs to be ignored - - refs = [] - - if (restricted): - ret = remote.push(("--mirror", "--dry-run")) - for info in ret: - # check if the local ref is either a head or a tag - if type(info.local_ref) in (git.refs.Head, git.refs.TagReference): - refs.append("".join(("+", info.local_ref.name, ":", info.remote_ref_string))) - - # do the push - - ret = None - if (refs): - ret = remote.push(refs) - else: - ret = remote.push("--mirror") - - # analyse the results and return success or failure - - for info in st: - if (info.flags & git.PushInfo.ERROR): - return False +import os + +def create(path, exists = True): + try: + repo = git.Repo(path) + except git.exc.NoSuchPathError: + git.Repo.init(path, bare = True) + return True + except git.exc.InvalidGitRepositoryError: + if not os.listdir(path): + git.Repo.init(path, bare = True) + return True + print("ERROR: The remote path is not a valid git repository.") + return False + if not repo.bare: + print("ERROR: The remote repository is not bare. Cannot push.") + return False + if not exists: + print("ERROR: The remote repository already exists.") + return exists + +def set_repo_description(path, desc): + descfile = os.path.join(path, "description") + if not os.path.isfile(descfile): + print("ERROR: Invalid or non-existent repository. Cannot find description file.") + return False + with open(descfile, "w") as f: + print(desc.strip(), file = f) return True diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index b7655ae..0000000 --- a/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -gitdb==0.6.4 -GitPython==2.0.5 -Logbook==0.12.5 -redis==2.10.5 -requests==2.10.0 -smmap==0.9.0 -tornado==4.3 diff --git a/server/CommandProtocol.py b/server/CommandProtocol.py deleted file mode 100644 index 405f6d5..0000000 --- a/server/CommandProtocol.py +++ /dev/null @@ -1,129 +0,0 @@ -# This file is part of Propagator, a KDE Sysadmin Project -# -# Copyright (C) 2015-2016 Boudhayan Gupta -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# 3. Neither the name of KDE e.V. (or its successor approved by the -# membership of KDE e.V.) nor the names of its contributors may be used -# to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR -# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# Protocol Description: -# -# create reponame - create reponame.git on server and mirrors -# move oldrepo newrepo - move/move oldrepo.git to newrepo.git -# update reponame - sync reponame.git with its mirrors -# delete reponame - delete reponame.git -# flush - try to commit all pending updates - -import os -import shlex - -from datetime import datetime -from collections import namedtuple -from ServerConfig import ServerConfig - -# task dispatcher -from RemoteControl import RemotePlugins -from RemoteDispatcher import RemoteDispatcher -dispatcher = RemoteDispatcher(RemotePlugins, "redis-experiment") - -# protocol exceptions - -class PropagatorProtocolException(Exception): - - def logline(self): - time = datetime.now().strftime("%Y-%m-%d %k:%M:%S") - return "{0} | {1}\n".format(time, str(self)) - -class InvalidCommandException(PropagatorProtocolException): - - def __init__(self, desc, command): - self.description = desc - self.command = command - super(InvalidCommandException, self).__init__("{0}: {1}".format(desc, command)) - -class InvalidActionException(PropagatorProtocolException): - - def __init__(self, action): - self.action = action - super(InvalidActionException, self).__init__("Invalid command: {0}".format(action)) - -# protocol parse helpers - -def ParseCommand(cmdString): - - components = shlex.split(cmdString) - action = components[0].lower() - - if not action in ("create", "move", "delete", "update"): - raise InvalidActionException(action) - ActionCommand = namedtuple("ActionCommand", ["action", "arguments", "upstream"]) - - if action == "create": - try: - args = { "srcRepo": components[1] } - except IndexError: - raise InvalidCommandException("create command does not contain source repository details", cmdString) - try: - upstream = components[2] - except IndexError: - upstream = None - elif action == "update": - try: - args = { "srcRepo": components[1] } - except IndexError: - raise InvalidCommandException("update command does not contain source repository details", cmdString) - try: - upstream = components[2] - except IndexError: - upstream = None - elif action == "delete": - try: - args = { "srcRepo": components[1] } - except IndexError: - raise InvalidCommandException("delete command does not contain source repository details", cmdString) - try: - upstream = components[2] - except IndexError: - upstream = None - elif action == "move": - try: - args = { "srcRepo": components[1], "destRepo": components[2] } - except IndexError: - raise InvalidCommandException("move command does not contain source and/or destination repository details", cmdString) - try: - upstream = components[3] - except IndexError: - upstream = None - return ActionCommand(action, args, upstream) - -def ExecuteCommand(context): - - if context.action == "create": - dispatcher.createRepo(context.arguments.get("srcRepo"), ident = context.upstream) - elif context.action == "update": - dispatcher.updateRepo(context.arguments.get("srcRepo"), ident = context.upstream) - elif context.action == "delete": - dispatcher.deleteRepo(context.arguments.get("srcRepo"), ident = context.upstream) - elif context.action == "move": - dispatcher.moveRepo(context.arguments.get("srcRepo"), context.arguments.get("destRepo"), ident = context.upstream) diff --git a/server/RemoteControl.py b/server/RemoteControl.py deleted file mode 100644 index c697cf0..0000000 --- a/server/RemoteControl.py +++ /dev/null @@ -1,97 +0,0 @@ -# This file is part of Propagator, a KDE Sysadmin Project -# -# Copyright (C) 2015-2016 Boudhayan Gupta -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# 3. Neither the name of KDE e.V. (or its successor approved by the -# membership of KDE e.V.) nor the names of its contributors may be used -# to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR -# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import os - -from importlib.machinery import SourceFileLoader -from logbook import Logger - -try: - import simplejson as json -except ImportError: - import json - -from ServerConfig import ServerConfig -from SyncJob import doSync - -class RemoteLoader(object): - - def __init__(self): - self.mLogger = Logger("RemoteLoader") - self.mLogger.info("loading remote management plugins...") - - defaultSearchPath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "remotes") - self.mPluginsSearchPaths = [ defaultSearchPath ] - self.mPluginsSearchPaths.extend(ServerConfig.get("RemotePluginsDir", [])) - - self.mLoadedPlugins = {} - self.mTaskMap = {} - - for searchPath in self.mPluginsSearchPaths: - self.mLogger.info(" loading plugins from directory: {0}", searchPath) - for pluginDir in os.listdir(searchPath): - pluginPath = os.path.join(searchPath, pluginDir) - pluginName, pluginEntry = self.loadPlugin(pluginPath) - if pluginName: - if pluginEntry["meta"].get("pushtype") == "restricted": - syncFunc = lambda src, dest: doSync(src, dest, True) - else: - syncFunc = lambda src, dest: doSync(src, dest, False) - self.mLoadedPlugins[pluginName] = pluginEntry - self.mTaskMap[pluginName] = pluginEntry.get("instance").createTaskMap(syncFunc) - self.mLogger.info(" loaded plugin: {0}".format(pluginName)) - self.mLogger.info("done loading remote management plugins") - - def loadPlugin(self, path): - metaFile = os.path.join(path, "metadata.json") - if not os.path.isfile(metaFile): - return (None, None) - - codeFile = os.path.join(path, "EntryPoint.py") - if not os.path.isfile(codeFile): - return (None, None) - - plugin = {} - with open(metaFile) as f: - plugin["meta"] = json.load(f) - pluginName = plugin.get("meta").get("name") - plugin["instance"] = SourceFileLoader("{0}.EntryPoint".format(pluginName), codeFile).load_module() - return (pluginName, plugin) - - def listLoadedPlugins(self): - return self.mLoadedPlugins.keys() - - def taskFunction(self, plugin, taskid): - if not plugin in self.mLoadedPlugins.keys(): - raise NotImplementedError("plugin {0} is not available".format(plugin)) - if not taskid in self.mTaskMap.get(plugin).keys(): - raise NotImplementedError("plugin {0} does not implement task {1}".format(plugin, taskid)) - return self.mTaskMap.get(plugin).get(taskid) - -RemotePlugins = RemoteLoader() diff --git a/server/RemoteDispatcher.py b/server/RemoteDispatcher.py deleted file mode 100644 index b7f95d4..0000000 --- a/server/RemoteDispatcher.py +++ /dev/null @@ -1,83 +0,0 @@ -# This file is part of Propagator, a KDE Sysadmin Project -# -# Copyright (C) 2015-2016 Boudhayan Gupta -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# 3. Neither the name of KDE e.V. (or its successor approved by the -# membership of KDE e.V.) nor the names of its contributors may be used -# to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR -# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from logbook import Logger -from redis import Redis -from uuid import uuid4 - -try: - import simplejson as json -except ImportError: - import json - -class RemoteDispatcher(object): - - def __init__(self, loader, qkey, host = "localhost", port = 6379, db = 0, password = None): - self.mLogger = Logger("RemoteDispatcher") - self.mLogger.info("connecting to the redis task queue...") - self.mRedisConn = Redis(host = host, port = port, db = db, password = password) - self.mQueueKey = "{}-IncomingTasks".format(qkey) - self.mLogger.info("connected") - self.mRemoteLoader = loader - - def createJob(self, plugin, jobclass, argsdict, depends = None): - jobid = "{0}-{1}".format(plugin, str(uuid4())) - payload = { - "jobclass": "{0}:{1}".format(plugin, jobclass), - "jobid": jobid, - "arguments": argsdict, - "depends": depends - } - self.mRedisConn.rpush(self.mQueueKey, json.dumps(payload)) - return jobid - - def createRepo(self, repo, desc = None, ifexists = False, ident = None): - for plugin in self.mRemoteLoader.listLoadedPlugins(): - createArgs = { "repo": repo, "desc": desc, "ifexists": ifexists } - self.createJob(plugin, "createrepo", createArgs) - - def setRepoDescription(self, repo, desc = None, ident = None): - for plugin in self.mRemoteLoader.listLoadedPlugins(): - descArgs = { "repo": repo, "desc": desc } - self.createJob(plugin, "setdesc", descArgs) - - def moveRepo(self, repo, dest, ident = None): - for plugin in self.mRemoteLoader.listLoadedPlugins(): - moveArgs = { "repo": repo, "dest": dest } - self.createJob(plugin, "moverepo", moveArgs) - - def updateRepo(self, repo, ident = None): - for plugin in self.mRemoteLoader.listLoadedPlugins(): - upArgs = { "repo": repo } - self.createjob(plugin, "updaterepo", upArgs) - - def deleteRepo(self, repo, ident = None): - for plugin in self.mRemoteLoader.listLoadedPlugins(): - delArgs = { "repo": repo } - self.createJob(plugin, "deleterepo", createArgs) diff --git a/server/ServerConfig.py b/server/ServerConfig.py deleted file mode 100644 index 01b0ea3..0000000 --- a/server/ServerConfig.py +++ /dev/null @@ -1,109 +0,0 @@ -# This file is part of Propagator, a KDE Sysadmin Project -# -# Copyright (C) 2015-2016 Boudhayan Gupta -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# 3. Neither the name of KDE e.V. (or its successor approved by the -# membership of KDE e.V.) nor the names of its contributors may be used -# to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR -# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import os -import collections - -try: - import simplejson as json -except ImportError: - import json - -class SyncedJSONDict(collections.UserDict): - - def __init__(self, filename, autoSync = False): - self.jsonFilename = filename - self.isDirty = False - self.autoSync = autoSync - - if os.path.isfile(self.jsonFilename): - self.syncRead() - - def syncWrite(self): - if self.isDirty: - with open(self.jsonFilename, "w") as f: - f.write(json.dumps(self.data, indent = 4, sort_keys = True)) - self.isDirty = False - - def syncRead(self): - with open(self.jsonFilename, "r") as f: - self.data = json.load(f) - self.isDirty = False - - def __setitem__(self, key, item): - super().__setitem__(key, item) - self.isDirty = True - if self.autoSync: - self.syncWrite() - - def __delitem__(self, key): - super().__delitem__(key, item) - self.isDirty = True - if self.autoSync: - self.syncWrite() - - def clear(self): - super().clear() - self.isDirty = True - if self.autoSync: - self.syncWrite() - - def pop(self, key, *args): - ret = super().pop(key, *args) - self.isDirty = True - if self.autoSync: - self.syncWrite() - return ret - - def popitem(self): - ret = super().popitem() - self.isDirty = True - if self.autoSync: - self.syncWrite() - return ret - - def update(self, dict = None): - if dict is None: - pass - elif isinstance(dict, UserDict.UserDict): - self.data = dict.data - self.isDirty = True - if self.autoSync: - self.syncWrite() - elif isinstance(dict, type({})): - self.data = dict - self.isDirty = True - if self.autoSync: - self.syncWrite() - else: - raise TypeError - -# load the main propagator configuration -ConfigPath = os.path.join(os.environ.get("GATOR_CONFIG_PATH"), "MasterConfig.json") -ServerConfig = SyncedJSONDict(ConfigPath) diff --git a/server/TaskServer.py b/server/TaskServer.py deleted file mode 100644 index cb7d81a..0000000 --- a/server/TaskServer.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/python3 -# This file is part of Propagator, a KDE Sysadmin Project -# -# Copyright 2015-2016 (C) Boudhayan Gupta -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# 3. Neither the name of KDE e.V. (or its successor approved by the -# membership of KDE e.V.) nor the names of its contributors may be used -# to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR -# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import sys -import os -import argparse -import traceback - -from redis import Redis -from logbook import Logger - -from RemoteControl import RemotePlugins -from ServerConfig import ServerConfig - -try: - import simplejson as json -except ImportError: - import json - -class RedisConsumer(object): - - def __init__(self, qkey, host = "localhost", port = 6379, db = 0, password = None): - self.mLogger = Logger(__class__.__name__) - self.mRedisConn = Redis(host = host, port = port, db = db, password = password) - self.mTaskQueueKey = "{}-IncomingTasks".format(qkey) - self.mFailedQueueKey = "{}-FailedTasks".format(qkey) - self.mDoneQueueKey = "{}-DoneTasks".format(qkey) - - self.mLogger.info("redis task queue keys are:") - self.mLogger.info(" incoming: {}".format(self.mTaskQueueKey)) - self.mLogger.info(" done: {}".format(self.mDoneQueueKey)) - self.mLogger.info(" failed: {}".format(self.mFailedQueueKey)) - - def runSingleTask(self): - queueKey, taskJson = (i.decode() for i in self.mRedisConn.blpop(self.mTaskQueueKey)) - task = json.loads(taskJson) - - plugin, taskid = task.get("jobclass").split(":") - try: - func = RemotePlugins.taskFunction(plugin, taskid) - ret = func(task.get("arguments")) - except Exception: - task["return"] = None - task["except"] = True - task["traceback"] = traceback.format_exc() - self.mLogger.exception() - self.mRedisConn.rpush(self.mFailedQueueKey, json.dumps(task)) - else: - task["return"] = ret - task["except"] = False - self.mRedisConn.rpush(self.mDoneQueueKey, json.dumps(task)) - - def runProcessLoop(self): - self.mLogger.info("consumer is now listening for tasks") - while True: self.runSingleTask() - -def CmdlineParse(): - - parser = argparse.ArgumentParser(prog = "TaskServer.py", - description = "Server to process tasks created by the propagator daemon") - parser.add_argument("-e", "--eid", dest = "eid", action = "store", default = os.getpid(), - help = "set an identifier for the server, for logging purposes (default is pid)") - return parser.parse_args() - -if __name__ == "__main__": - - # parse command line arguments - info = CmdlineParse() - - # set up logging - from logbook import StreamHandler, Logger - StreamHandler(sys.stdout).push_application() - logger = Logger("TaskServer-{}".format(info.eid)) - - # start up - logger.info("starting...") - from RemoteControl import RemotePlugins - consumer = RedisConsumer(ServerConfig.get("RedisQueueKey", "GatorDefault")) - consumer.runProcessLoop() diff --git a/server/remotes/anongit/EntryPoint.py b/server/remotes/anongit/EntryPoint.py deleted file mode 100644 index 7649fa4..0000000 --- a/server/remotes/anongit/EntryPoint.py +++ /dev/null @@ -1,2 +0,0 @@ -def createTaskMap(hello): - return {} diff --git a/server/remotes/anongit/metadata.json b/server/remotes/anongit/metadata.json deleted file mode 100644 index c9f5030..0000000 --- a/server/remotes/anongit/metadata.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "anongit", - "description": "Remote management plugin for unmanaged anonymous read-only git servers", - "version": "1.0.0", - "license": "BSD", - "author": "Boudhayan Gupta ", - - "pushtype": "full", - "identspec": "anongit", - "configvar": "GATOR_PCFG_ANONGIT" -} diff --git a/server/remotes/github/EntryPoint.py b/server/remotes/github/EntryPoint.py deleted file mode 100644 index ce22a47..0000000 --- a/server/remotes/github/EntryPoint.py +++ /dev/null @@ -1,157 +0,0 @@ -# This file is part of Propagator, a KDE Sysadmin Project -# -# Copyright (C) 2015-2016 Boudhayan Gupta -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# 3. Neither the name of KDE e.V. (or its successor approved by the -# membership of KDE e.V.) nor the names of its contributors may be used -# to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR -# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import re -import os -import sys -import requests - -try: - import simplejson as json -except ImportError: - import json - -class GithubPlugin(object): - - def __init__(self, sfunc): - cfgPath = os.path.join(os.environ.get("GATOR_CONFIG_PATH"), "RemotesGithub.json") - with open(cfgPath) as f: - cfgDict = json.load(f) - self.mAccessToken = cfgDict.get("accesstoken") - self.mGithubOrg = cfgDict.get("organization") - self.mExceptChecks = [ re.compile(i) for i in cfgDict.get("excepts") ] - - mcfgPath = os.path.join(os.environ.get("GATOR_CONFIG_PATH"), "MasterConfig.json") - with open(mcfgPath) as f: - self.mRepoBase = json.load(f).get("RepoRoot") - - self.mSession = requests.Session() - self.mSession.headers.update({"Accept": "application/vnd.github.v3+json"}) - self.mSession.headers.update({"Authorization": " ".join(("token", self.mAccessToken))}) - - self.mSyncFunc = sfunc - self.mReposEndpoint = "https://api.github.com/repos" - self.mOrgsEndpoint = "https://api.github.com/orgs" - self.mTaskMap = { - "create": self.createRepo, - "move": self.moveRepo, - "update": self.syncRepo, - "delete": self.deleteRepo, - "setdesc": self.setRepoDescription - } - - def __repr__(self): - return "".format(self.mGithubOrg) - - def keys(self): - return self.mTaskMap.keys() - - def get(self, key): - return self.mTaskMap.get(key) - - def stripRepoName(self, name): - if name.endswith(".git"): - return name[:-4] - return name - - def repoExists(self, name): - url = "{0}/{1}/{2}".format(self.mReposEndpoint, self.mGithubOrg, name) - r = self.mSession.get(url) - return ((r.ok) and ("id" in r.json.keys())) - - def createRepo(self, args): - reponame = args.get("repo") - for i in self.mExceptChecks: - if i.match(reponame): - return True - reponame = self.stripRepoName(reponame) - - # if the repo already exists and create if exists is false, exit early - if not args.get("ifexists") and self.repoExists(reponame): - return True - - # build up the create request and execute it - payload = { - "name": reponame, - "description": args.get("desc", "This repository has no description"), - "private": False, - "has_issues": False, - "has_wiki": False, - "has_downloads": False, - "auto_init": False, - } - url = "{0}/{1}/{2}".format(self.mOrgsEndpoint, self.mGithubOrg, "repos") - r = self.mSession.post(url, data = json.dumps(payload)) - return ((r.status_code == 201) and ("id" in r.json.keys())) - - def moveRepo(self, args): - oldname = args.get("repo") - for i in self.mExceptChecks: - if i.match(oldname): - return True - oldname = self.stripRepoName(oldname) - newname = self.stripRepoName(args.get("dest")) - payload = { "name": newname } - url = "{0}/{1}/{2}".format(self.mReposEndpoint, self.mGithubOrg, oldname) - r = self.mSession.patch(url, data = json.dumps(payload)) - return ((r.status_code == 201) and ("id" in r.json.keys())) - - def syncRepo(self, args): - reponame = args.get("repo") - for i in self.mExceptChecks: - if i.match(reponame): - return True - srcdir = os.path.join(self.mRepoBase, reponame) - desturl = "git@github.com:{0}/{1}".format(self.mGithubOrg, reponame) - self.createRepo(args) - return self.mSyncFunc(srcdir, desturl) - - def deleteRepo(self, args): - reponame = args.get("repo") - for i in self.mExceptChecks: - if i.match(reponame): - return True - reponame = self.stripRepoName(reponame) - url = "{0}/{1}/{2}".format(self.mReposEndpoint, self.mGithubOrg, reponame) - r = self.mSession.delete(url) - return (r.status_code == 204) - - def setRepoDescription(self, args): - reponame = args.get("repo") - for i in self.mExceptChecks: - if i.match(reponame): - return True - reponame = self.stripRepoName(reponame) - payload = { "description": desc } - url = "{0}/{1}/{2}".format(self.mReposEndpoint, self.mGithubOrg, reponame) - r = self.mSession.patch(url, data = json.dumps(payload)) - return ((r.status_code == 201) and ("id" in r.json.keys())) - -def createTaskMap(sfunc): - return GithubPlugin(sfunc) diff --git a/server/remotes/github/metadata.json b/server/remotes/github/metadata.json deleted file mode 100644 index 41239ce..0000000 --- a/server/remotes/github/metadata.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name": "github", - "description": "Remote management plugin for GitHub Organizations", - "version": "1.0.0", - "license": "BSD", - "author": "Boudhayan Gupta ", - "pushtype": "restricted" -} diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..1584992 --- /dev/null +++ b/setup.py @@ -0,0 +1,25 @@ +from setuptools import setup, find_packages + +setup( + name = "propagator", + version = "1.0.0", + author = "Boudhayan Gupta", + author_email = "bgupta@kde.org", + description = ("A git mirror fleet manager"), + license = "BSD", + keywords = "git mirror devops", + url = "http://www.kde.org/", + packages = find_packages(), + install_requires = ( + "GitPython", + ), + classifiers = ( + "Development Status :: 4 - Beta", + "License :: OSI Approved :: BSD License", + ), + entry_points = { + "console_scripts": ( + "propagator-agent = propagator.agent:main", + ), + }, +) diff --git a/supervisord.conf b/supervisord.conf deleted file mode 100644 index 2ed7315..0000000 --- a/supervisord.conf +++ /dev/null @@ -1,23 +0,0 @@ -[supervisord] -childlogdir = %(here)s/logs/ -logfile = %(here)s/logs/supervisord.log -logfile_backups = 10 -loglevel = info -nocleanup = true -nodaemon = true -environment = GATOR_CONFIG_PATH="%(here)s/config" - -[program:taskserver] -command = python3 %(here)s/server/TaskServer.py --eid %(process_num)s -directory = %(here)s/server -process_name = TaskServer-%(process_num)s -numprocs = 4 -stdout_logfile = %(here)s/logs/TaskServer-%(process_num)s.log -redirect_stderr = true - -#[program:server] -#command = python3 %(here)s/server/Server.py -#directory = %(here)s/server -#process_name = server -#stdout_logfile = %(here)s/logs/server.log -#redirect_stderr = true