diff --git a/config/MasterConfig.json b/config/MasterConfig.json index 192e4ea..4d40948 100644 --- a/config/MasterConfig.json +++ b/config/MasterConfig.json @@ -1,29 +1,8 @@ { "ServerListenIP": "::1", "ServerListenPort": 58192, "ControlListenIP": "::1", "ControlListenPort": 58193, - "RepoRoot": "/home/bg14ina/testRepos/", - - "GithubEnabled": false, - "GithubPrefix": "kde", - "GithubUser": "git", - "GithubOrganization": "kde", - "GithubAPIKeyFile": "", - - "AnongitEnabled": false, - "AnongitPrefix": "repos", - "AnongitUser": "git", - "AnongitAPIKeyFile": "~/GatorAgent", - "AnongitServers": [ - "89.248.108.131" - ], - - "GithubExcepts": [ - "gitolite-admin.git", - "scratch/*", - "websites/*", - "clones/*" - ], - "AnongitExcepts": [] + "RedisQueueKey": "GatorQueue", + "RepoRoot": "/home/bg14ina/testRepos/" } diff --git a/config/RemotesGithub.json b/config/RemotesGithub.json index 67b6323..130cbb8 100644 --- a/config/RemotesGithub.json +++ b/config/RemotesGithub.json @@ -1,4 +1,9 @@ { "organization": "BaloneyGeekCorp", - "accesstoken": "dummytoken" + "accesstoken": "dummytoken", + + "excepts": [ + "gitolite-admin.git", + "([a-zA-Z0-9]*)/(.*)" + ] } diff --git a/server/AnongitRemote.py b/server/AnongitRemote.py deleted file mode 100644 index 1be4d1f..0000000 --- a/server/AnongitRemote.py +++ /dev/null @@ -1,103 +0,0 @@ -# This file is part of Propagator, a KDE Sysadmin Project -# -# Copyright 2015 Boudhayan Gupta -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# 3. Neither the name of KDE e.V. (or its successor approved by the -# membership of KDE e.V.) nor the names of its contributors may be used -# to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR -# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import subprocess -import os - -try: - import simplejson as json -except ImportError: - import json - -class AnongitRemote(object): - - def __init__(self, name, host, desc = "This repository has no description"): - - self.HOST = host - self.REPO_NAME = name - self.REPO_DESC = desc - - # load the ssh configuration - - cfgPath = os.environ.get("GATOR_CONFIG_FILE") - cfgData = {} - with open(cfgPath) as f: - cfgData = json.load(f) - - self.SSH_USER = cfgData.get("AnongitUser") - self.SSH_KEYFILE = os.path.expanduser(cfgData.get("AnongitAPIKeyFile")) - - def __repr__(self): - - return ("" % (self.HOST, self.REPO_NAME)) - - def __runSshCommand(self, command): - - hostString = "%s@%s" % (self.SSH_USER, self.HOST) - cmdContext = ("ssh", "-i", self.SSH_KEYFILE, hostString, command) - ssh = subprocess.Popen(cmdContext, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = False) - result = [i.decode().strip() for i in ssh.stdout.readlines()] - - if "OK" in result: - return True - result = ssh.stderr.readlines() - return False - - def setRepoDescription(self, desc): - - self.REPO_DESC = desc - if self.repoExists(): - command = "SETDESC %s \"%s\"" % (self.REPO_NAME, self.REPO_DESC) - return self.__runSshCommand(command) - return True - - def repoExists(self): - - command = "EXISTS %s" % (self.REPO_NAME) - return self.__runSshCommand(command) - - def createRepo(self): - - command = "CREATE %s \"%s\"" % (self.REPO_NAME, self.REPO_DESC) - return self.__runSshCommand(command) - - def deleteRepo(self): - - command = "DELETE %s" % (self.REPO_NAME) - return self.__runSshCommand(command) - - def moveRepo(self, newname): - - command = "MOVE %s %s" % (self.REPO_NAME, newname) - ret = self.__runSshCommand(command) - - if ret: - self.REPO_NAME = newname - return True - return False diff --git a/server/TaskServer.py b/server/TaskServer.py index 4127a7a..cb7d81a 100644 --- a/server/TaskServer.py +++ b/server/TaskServer.py @@ -1,101 +1,106 @@ #!/usr/bin/python3 # This file is part of Propagator, a KDE Sysadmin Project # # Copyright 2015-2016 (C) Boudhayan Gupta # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # 3. Neither the name of KDE e.V. (or its successor approved by the # membership of KDE e.V.) nor the names of its contributors may be used # to endorse or promote products derived from this software without # specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import sys import os import argparse import traceback from redis import Redis from logbook import Logger from RemoteControl import RemotePlugins +from ServerConfig import ServerConfig try: import simplejson as json except ImportError: import json class RedisConsumer(object): def __init__(self, qkey, host = "localhost", port = 6379, db = 0, password = None): - self.mLogger = Logger("RedisConsumer") + self.mLogger = Logger(__class__.__name__) self.mRedisConn = Redis(host = host, port = port, db = db, password = password) self.mTaskQueueKey = "{}-IncomingTasks".format(qkey) self.mFailedQueueKey = "{}-FailedTasks".format(qkey) self.mDoneQueueKey = "{}-DoneTasks".format(qkey) + self.mLogger.info("redis task queue keys are:") + self.mLogger.info(" incoming: {}".format(self.mTaskQueueKey)) + self.mLogger.info(" done: {}".format(self.mDoneQueueKey)) + self.mLogger.info(" failed: {}".format(self.mFailedQueueKey)) + def runSingleTask(self): queueKey, taskJson = (i.decode() for i in self.mRedisConn.blpop(self.mTaskQueueKey)) task = json.loads(taskJson) plugin, taskid = task.get("jobclass").split(":") try: func = RemotePlugins.taskFunction(plugin, taskid) ret = func(task.get("arguments")) except Exception: task["return"] = None task["except"] = True task["traceback"] = traceback.format_exc() self.mLogger.exception() self.mRedisConn.rpush(self.mFailedQueueKey, json.dumps(task)) else: task["return"] = ret task["except"] = False self.mRedisConn.rpush(self.mDoneQueueKey, json.dumps(task)) def runProcessLoop(self): self.mLogger.info("consumer is now listening for tasks") while True: self.runSingleTask() def CmdlineParse(): - parser = argparse.ArgumentParser(prog = "TaskServer.py", description = "Server to process tasks created by the propagator daemon") - parser.add_argument("-e", "--eid", dest = "eid", action = "store", default = os.getpid(), help = "set an identifier for the server, for logging purposes (default is pid)") - + parser = argparse.ArgumentParser(prog = "TaskServer.py", + description = "Server to process tasks created by the propagator daemon") + parser.add_argument("-e", "--eid", dest = "eid", action = "store", default = os.getpid(), + help = "set an identifier for the server, for logging purposes (default is pid)") return parser.parse_args() if __name__ == "__main__": # parse command line arguments info = CmdlineParse() # set up logging from logbook import StreamHandler, Logger StreamHandler(sys.stdout).push_application() logger = Logger("TaskServer-{}".format(info.eid)) # start up logger.info("starting...") from RemoteControl import RemotePlugins - print(RemotePlugins.listLoadedPlugins()) - - consumer = RedisConsumer("redis-experiment") + consumer = RedisConsumer(ServerConfig.get("RedisQueueKey", "GatorDefault")) consumer.runProcessLoop() diff --git a/supervisord.conf b/supervisord.conf index 9d512c2..2ed7315 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -1,23 +1,23 @@ [supervisord] childlogdir = %(here)s/logs/ logfile = %(here)s/logs/supervisord.log logfile_backups = 10 loglevel = info nocleanup = true nodaemon = true -environment = GATOR_CONFIG_FILE="%(here)s/config/ServerConfig.json" +environment = GATOR_CONFIG_PATH="%(here)s/config" -[program:worker] -command = celery worker -A CeleryWorkers --loglevel=INFO -n worker-%(process_num)s +[program:taskserver] +command = python3 %(here)s/server/TaskServer.py --eid %(process_num)s directory = %(here)s/server -process_name = worker-%(process_num)s +process_name = TaskServer-%(process_num)s numprocs = 4 -stdout_logfile = %(here)s/logs/worker-%(process_num)s.log +stdout_logfile = %(here)s/logs/TaskServer-%(process_num)s.log redirect_stderr = true -[program:server] -command = python3 %(here)s/server/Server.py -directory = %(here)s/server -process_name = server -stdout_logfile = %(here)s/logs/server.log -redirect_stderr = true +#[program:server] +#command = python3 %(here)s/server/Server.py +#directory = %(here)s/server +#process_name = server +#stdout_logfile = %(here)s/logs/server.log +#redirect_stderr = true