diff --git a/README.md b/README.md index d46389a..bd6b94c 100644 --- a/README.md +++ b/README.md @@ -1,61 +1,62 @@ # wikitolearn-sdk This repository contains the dev kit for the WikiToLearn's microservices-oriented stack. These tools are helpful to work with the stack in a "point and click" fashion or the nearest approximation possible. ## Setup **Minimum requirements:** * git * docker * docker-compose * python3 * python3-argcomplete * python3-yaml To setup the WikiToLearn dev kit you have to: -1. clone this repository -2. create the `config/repositories.yml` file (an example is `config/repositories.example.yml`) -3. create the `config/config.yml` file (an example is `config/config.example.yml`) -4. run `$ source ./setup-env` -5. run `$ wtl-setup-test` to test if you have fulfilled the requirements and the configurations +1. setup the `kde:` prefix in git with [this](https://community.kde.org/Sysadmin/GitKdeOrgManual#Let_Git_rewrite_URL_prefixes) guide +2. clone this repository +3. (optional) create the `config/repositories.yml` file (an example is `config/repositories.example.yml`) +4. create the `config/config.yml` file (an example is `config/config.example.yml`) +5. run `$ source ./setup-env` +6. run `$ wtl-setup-test` to test if you have fulfilled the requirements and the configurations ## How to manage services ### Build To build the services you have to run: `$ wtl-services-build` This command clone/pull the repositories and build all the services defined into docker-compose files. ### Run To run the services you have to run: `$ wtl-services-run` This command start all the services. ### Data restore If you need to restore a dump you can run: `$ wtl-dump-load` N.B.: a `dump_url` must be set within `config/config.yml` file and the services have to be started. ### Stop To stop the services you have to run: $ wtl-services-stop` This command stop and remove the service containers. ## A note about the repositories directory WikiToLearn's repositories will be cloned into the `repositories/` directory. This is a non-configurable option. It allows tools to work with the assumption of the relative location of each repository. ## LICENSE See [LICENSE](LICENSE) file. diff --git a/bin/wtl-dummy-dataset b/bin/wtl-dummy-dataset new file mode 100755 index 0000000..9c177f2 --- /dev/null +++ b/bin/wtl-dummy-dataset @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import requests +import os +import time +import argcomplete, argparse +import yaml +import subprocess +from pprint import pprint +import sys + +print("This scirpt doesn't work") +sys.exit(1) + +WTL_DEV_KIT_PATH = os.environ.get("WTL_DEV_KIT_PATH") +WTL_DEV_KIT_REPOS_PATH = os.environ.get("WTL_DEV_KIT_REPOS_PATH") +WTL_DEV_KIT_BIN_PATH = os.environ.get("WTL_DEV_KIT_BIN_PATH") + +config = {} + +with open(WTL_DEV_KIT_PATH + "/config/config.yml") as fh: + config['config'] = yaml.load(fh) + +if os.path.isfile(WTL_DEV_KIT_PATH + "/config/repositories.yml"): + with open(WTL_DEV_KIT_PATH + "/config/repositories.yml") as fh: + config['repositories'] = yaml.load(fh) +else: + with open(WTL_DEV_KIT_PATH + "/config/repositories.example.yml") as fh: + config['repositories'] = yaml.load(fh) + +parser = argparse.ArgumentParser() + +argcomplete.autocomplete(parser) +args = parser.parse_args() + +os.environ['KEYCLOAK_URI'] = 'http://{host_ip}:9080'.format(host_ip=config['config']['host_ip']) +os.environ['COURSES_BACKEND_URI'] = 'http://{host_ip}:10000'.format(host_ip=config['config']['host_ip']) +os.environ['CHAPTERS_BACKEND_URI'] = 'http://{host_ip}:10001'.format(host_ip=config['config']['host_ip']) +os.environ['PAGES_BACKEND_URI'] = 'http://{host_ip}:10002'.format(host_ip=config['config']['host_ip']) +os.environ['COURSE_MIDTIER_URI'] = 'http://{host_ip}:11000'.format(host_ip=config['config']['host_ip']) +os.environ['PWA_GATEWAY_URI'] = 'http://{host_ip}:12000'.format(host_ip=config['config']['host_ip']) +os.environ['PUBLIC_KEYCLOAK_URI'] = 'http://{host_ip}:9080'.format(host_ip=config['config']['host_ip']) +os.environ['PUBLIC_PWA_GATEWAY_URI'] = 'http://{host_ip}:12000'.format(host_ip=config['config']['host_ip']) + + +# keycloak config +keycloak_realm_id = "wikitolearn-local" +keycloak_realm_username = "admin" +keycloak_realm_password = "admin" +keycloak_base_url = os.environ.get("KEYCLOAK_URI") + "/auth/" +keycloak_realm_base_url = keycloak_base_url + "admin/realms/{}".format(keycloak_realm_id) + +keycloak_auth_obj = None +keycloak_last_login_success = 0 +def keycloak_get_auth_headers(base_headers={}): + global keycloak_auth_obj + global keycloak_base_url + global keycloak_last_login_success + headers = dict(base_headers) + do_login = False + if keycloak_auth_obj == None: + do_login = True + else: + if time.time() - keycloak_last_login_success > (keycloak_auth_obj['expires_in'] - 5): + do_login = True + + if do_login: + auth_endpoint = keycloak_base_url + "realms/master/" + data = {} + data["client_id"] = "admin-cli" + data["username"] = keycloak_realm_username + data["password"] = keycloak_realm_password + data["grant_type"] = "password" + + try: + auth_r = requests.post( + auth_endpoint + "protocol/openid-connect/token", + data=data + ) + auth_r.raise_for_status() + keycloak_auth_obj = auth_r.json() + keycloak_last_login_success = time.time() + except Exception as e: + print("Keycloak Login Exception") + raise e + if keycloak_auth_obj != None and 'access_token' in keycloak_auth_obj: + headers["Authorization"] = "Bearer {}".format(keycloak_auth_obj['access_token']) + else: + pprint(keycloak_auth_obj) + time.sleep(1) + return headers + +if requests.get(keycloak_realm_base_url,headers=keycloak_get_auth_headers()).status_code == 404: + print("create realm") + create_realm = requests.post( + keycloak_base_url + "admin/realms", + json={ + 'enabled': True, + 'id': keycloak_realm_id, + 'realm': keycloak_realm_id + }, + headers=keycloak_get_auth_headers() + ) + create_realm.raise_for_status() + +has_client_frontend = False + +clients_list_request = requests.get(keycloak_realm_base_url + "/clients",headers=keycloak_get_auth_headers()) +for client in clients_list_request.json(): + has_client_frontend = has_client_frontend or (client.get('clientId') == "frontend") + +if not has_client_frontend: + print("create client frontend") + create_frontend_client = requests.post( + keycloak_realm_base_url + "/clients", + json={ + "enabled":True, + "attributes":{}, + "redirectUris":[], + "clientId":"frontend", + "rootUrl":"http://localhost:13000", + "protocol":"openid-connect" + }, + headers=keycloak_get_auth_headers() + ) + create_frontend_client.raise_for_status() + +for user_n in range(0, 10): + keycloak_user = {} + keycloak_user['username'] = "user{}".format(user_n) + keycloak_user['firstName'] = "firstName" + "user{}".format(user_n) + keycloak_user['lastName'] = "lastName" + "user{}".format(user_n) + keycloak_user['email'] = "user{}".format(user_n) + "@localhost" + keycloak_user['credentials'] = [ + { + "type": "password", + "value": "password", + } + ] + keycloak_user["enabled"] = True + keycloak_user["emailVerified"] = True + keycloak_user["attributes"] = { + "mw_id": user_n, + } + + users_endpoint = keycloak_realm_base_url + "/users" + create_user_request = requests.post( + users_endpoint, + json=keycloak_user, + headers=keycloak_get_auth_headers() + ) + try: + create_user_request.raise_for_status() + except Exception as exc: + print(exc) + +courses_index_request = requests.get(os.environ.get("COURSES_BACKEND_URI") + "/v1/courses") +pprint(courses_index_request.json()) diff --git a/bin/wtl-migration-dump-load b/bin/wtl-migration-dump-load index a126b43..ef0dc19 100755 --- a/bin/wtl-migration-dump-load +++ b/bin/wtl-migration-dump-load @@ -1,55 +1,59 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import os import argcomplete, argparse import yaml import subprocess from pprint import pprint WTL_DEV_KIT_PATH = os.environ.get("WTL_DEV_KIT_PATH") WTL_DEV_KIT_REPOS_PATH = os.environ.get("WTL_DEV_KIT_REPOS_PATH") WTL_DEV_KIT_BIN_PATH = os.environ.get("WTL_DEV_KIT_BIN_PATH") config = {} with open(WTL_DEV_KIT_PATH + "/config/config.yml") as fh: config['config'] = yaml.load(fh) -with open(WTL_DEV_KIT_PATH + "/config/repositories.yml") as fh: - config['repositories'] = yaml.load(fh) +if os.path.isfile(WTL_DEV_KIT_PATH + "/config/repositories.yml"): + with open(WTL_DEV_KIT_PATH + "/config/repositories.yml") as fh: + config['repositories'] = yaml.load(fh) +else: + with open(WTL_DEV_KIT_PATH + "/config/repositories.example.yml") as fh: + config['repositories'] = yaml.load(fh) parser = argparse.ArgumentParser() argcomplete.autocomplete(parser) args = parser.parse_args() tmp_dir = WTL_DEV_KIT_PATH + "/tmp/" subprocess.call([ "wget", "-N", config['config']['migration']['dump_url'], ], cwd=tmp_dir) subprocess.call([ "rm", "-Rfv", "last_output", ], cwd=tmp_dir) subprocess.call([ "mkdir", "last_output", ], cwd=tmp_dir) subprocess.call([ "tar", "-xvzf", "last_output.tar.gz", "-C", "last_output", ], cwd=tmp_dir) subprocess.call([ "./restore-dump.sh", ], cwd=tmp_dir + "/last_output/") diff --git a/bin/wtl-services-build b/bin/wtl-services-build index f8ea1db..02b2d6b 100755 --- a/bin/wtl-services-build +++ b/bin/wtl-services-build @@ -1,73 +1,78 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import os import argcomplete, argparse import yaml import subprocess from pprint import pprint WTL_DEV_KIT_PATH = os.environ.get("WTL_DEV_KIT_PATH") WTL_DEV_KIT_REPOS_PATH = os.environ.get("WTL_DEV_KIT_REPOS_PATH") WTL_DEV_KIT_BIN_PATH = os.environ.get("WTL_DEV_KIT_BIN_PATH") config = {} with open(WTL_DEV_KIT_PATH + "/config/config.yml") as fh: config['config'] = yaml.load(fh) -with open(WTL_DEV_KIT_PATH + "/config/repositories.yml") as fh: - config['repositories'] = yaml.load(fh) +if os.path.isfile(WTL_DEV_KIT_PATH + "/config/repositories.yml"): + with open(WTL_DEV_KIT_PATH + "/config/repositories.yml") as fh: + config['repositories'] = yaml.load(fh) +else: + with open(WTL_DEV_KIT_PATH + "/config/repositories.example.yml") as fh: + config['repositories'] = yaml.load(fh) parser = argparse.ArgumentParser() argcomplete.autocomplete(parser) args = parser.parse_args() os.environ['KEYCLOAK_URI'] = 'http://{host_ip}:9080'.format(host_ip=config['config']['host_ip']) os.environ['COURSES_BACKEND_URI'] = 'http://{host_ip}:10000'.format(host_ip=config['config']['host_ip']) os.environ['CHAPTERS_BACKEND_URI'] = 'http://{host_ip}:10001'.format(host_ip=config['config']['host_ip']) os.environ['PAGES_BACKEND_URI'] = 'http://{host_ip}:10002'.format(host_ip=config['config']['host_ip']) os.environ['COURSE_MIDTIER_URI'] = 'http://{host_ip}:11000'.format(host_ip=config['config']['host_ip']) os.environ['PWA_GATEWAY_URI'] = 'http://{host_ip}:12000'.format(host_ip=config['config']['host_ip']) os.environ['PUBLIC_KEYCLOAK_URI'] = 'http://localhost:9080' os.environ['PUBLIC_PWA_GATEWAY_URI'] = 'http://localhost:12000' for service_name in [ "shared-services", "courses-backend", "chapters-backend", "pages-backend", "coursessecurity-backend", "course-midtier", "pwa-gateway", "frontend", ]: if service_name in config['repositories']: service_path = WTL_DEV_KIT_REPOS_PATH + "/" + service_name print("Build '{service_name}'".format(service_name=service_name)) if not os.path.isdir(service_path): clone_cmd = [ "git", "clone", config['repositories'][service_name]['git'], service_path ] if 'git_branch' in config['repositories'][service_name]: clone_cmd.append("-b") clone_cmd.append(config['repositories'][service_name]['git_branch']) subprocess.call(clone_cmd) subprocess.call([ "git", "pull", ], cwd=service_path) docker_compose_command = [ "docker-compose", "-f","docker-compose.yml", ] if os.path.isfile(service_path + "/" + "docker-compose-dev-deps.yml"): docker_compose_command.append("-f") docker_compose_command.append("docker-compose-dev-deps.yml") docker_compose_command.append("build") - subprocess.call(docker_compose_command, cwd=service_path, env=os.environ.copy()) + if subprocess.call(docker_compose_command, cwd=service_path, env=os.environ.copy()) != 0: + raise Exception("{} failed".format(docker_compose_command)) diff --git a/bin/wtl-services-run b/bin/wtl-services-run index dd00589..e4cbae8 100755 --- a/bin/wtl-services-run +++ b/bin/wtl-services-run @@ -1,58 +1,62 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import os import argcomplete, argparse import yaml import subprocess from pprint import pprint WTL_DEV_KIT_PATH = os.environ.get("WTL_DEV_KIT_PATH") WTL_DEV_KIT_REPOS_PATH = os.environ.get("WTL_DEV_KIT_REPOS_PATH") WTL_DEV_KIT_BIN_PATH = os.environ.get("WTL_DEV_KIT_BIN_PATH") config = {} with open(WTL_DEV_KIT_PATH + "/config/config.yml") as fh: config['config'] = yaml.load(fh) -with open(WTL_DEV_KIT_PATH + "/config/repositories.yml") as fh: - config['repositories'] = yaml.load(fh) +if os.path.isfile(WTL_DEV_KIT_PATH + "/config/repositories.yml"): + with open(WTL_DEV_KIT_PATH + "/config/repositories.yml") as fh: + config['repositories'] = yaml.load(fh) +else: + with open(WTL_DEV_KIT_PATH + "/config/repositories.example.yml") as fh: + config['repositories'] = yaml.load(fh) parser = argparse.ArgumentParser() argcomplete.autocomplete(parser) args = parser.parse_args() os.environ['KEYCLOAK_URI'] = 'http://{host_ip}:9080'.format(host_ip=config['config']['host_ip']) os.environ['COURSES_BACKEND_URI'] = 'http://{host_ip}:10000'.format(host_ip=config['config']['host_ip']) os.environ['CHAPTERS_BACKEND_URI'] = 'http://{host_ip}:10001'.format(host_ip=config['config']['host_ip']) os.environ['PAGES_BACKEND_URI'] = 'http://{host_ip}:10002'.format(host_ip=config['config']['host_ip']) os.environ['COURSE_MIDTIER_URI'] = 'http://{host_ip}:11000'.format(host_ip=config['config']['host_ip']) os.environ['PWA_GATEWAY_URI'] = 'http://{host_ip}:12000'.format(host_ip=config['config']['host_ip']) os.environ['PUBLIC_KEYCLOAK_URI'] = 'http://localhost:9080' os.environ['PUBLIC_PWA_GATEWAY_URI'] = 'http://localhost:12000' for service_name in [ "shared-services", "courses-backend", "chapters-backend", "pages-backend", "coursessecurity-backend", "course-midtier", "pwa-gateway", "frontend", ]: if service_name in config['repositories']: service_path = WTL_DEV_KIT_REPOS_PATH + "/" + service_name print("Run '{service_name}'".format(service_name=service_name)) docker_compose_command = [ "docker-compose", "-f","docker-compose.yml", ] if os.path.isfile(service_path + "/" + "docker-compose-dev-deps.yml"): docker_compose_command.append("-f") docker_compose_command.append("docker-compose-dev-deps.yml") docker_compose_command.append("up") docker_compose_command.append("-d") subprocess.call(docker_compose_command, cwd=service_path) diff --git a/bin/wtl-services-stop b/bin/wtl-services-stop index 85d0694..c1158c9 100755 --- a/bin/wtl-services-stop +++ b/bin/wtl-services-stop @@ -1,57 +1,61 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import os import argcomplete, argparse import yaml import subprocess from pprint import pprint WTL_DEV_KIT_PATH = os.environ.get("WTL_DEV_KIT_PATH") WTL_DEV_KIT_REPOS_PATH = os.environ.get("WTL_DEV_KIT_REPOS_PATH") WTL_DEV_KIT_BIN_PATH = os.environ.get("WTL_DEV_KIT_BIN_PATH") config = {} with open(WTL_DEV_KIT_PATH + "/config/config.yml") as fh: config['config'] = yaml.load(fh) -with open(WTL_DEV_KIT_PATH + "/config/repositories.yml") as fh: - config['repositories'] = yaml.load(fh) +if os.path.isfile(WTL_DEV_KIT_PATH + "/config/repositories.yml"): + with open(WTL_DEV_KIT_PATH + "/config/repositories.yml") as fh: + config['repositories'] = yaml.load(fh) +else: + with open(WTL_DEV_KIT_PATH + "/config/repositories.example.yml") as fh: + config['repositories'] = yaml.load(fh) parser = argparse.ArgumentParser() argcomplete.autocomplete(parser) args = parser.parse_args() os.environ['KEYCLOAK_URI'] = 'http://{host_ip}:9080'.format(host_ip=config['config']['host_ip']) os.environ['COURSES_BACKEND_URI'] = 'http://{host_ip}:10000'.format(host_ip=config['config']['host_ip']) os.environ['CHAPTERS_BACKEND_URI'] = 'http://{host_ip}:10001'.format(host_ip=config['config']['host_ip']) os.environ['PAGES_BACKEND_URI'] = 'http://{host_ip}:10002'.format(host_ip=config['config']['host_ip']) os.environ['COURSE_MIDTIER_URI'] = 'http://{host_ip}:11000'.format(host_ip=config['config']['host_ip']) os.environ['PWA_GATEWAY_URI'] = 'http://{host_ip}:12000'.format(host_ip=config['config']['host_ip']) os.environ['PUBLIC_KEYCLOAK_URI'] = 'http://localhost:9080' os.environ['PUBLIC_PWA_GATEWAY_URI'] = 'http://localhost:12000' for service_name in reversed([ "shared-services", "courses-backend", "chapters-backend", "pages-backend", "coursessecurity-backend", "course-midtier", "pwa-gateway", "frontend", ]): if service_name in config['repositories']: service_path = WTL_DEV_KIT_REPOS_PATH + "/" + service_name print("Run '{service_name}'".format(service_name=service_name)) docker_compose_command = [ "docker-compose", "-f","docker-compose.yml", ] if os.path.isfile(service_path + "/" + "docker-compose-dev-deps.yml"): docker_compose_command.append("-f") docker_compose_command.append("docker-compose-dev-deps.yml") docker_compose_command.append("down") subprocess.call(docker_compose_command, cwd=service_path) diff --git a/bin/wtl-setup-test b/bin/wtl-setup-test index ec748fd..8ac93ce 100755 --- a/bin/wtl-setup-test +++ b/bin/wtl-setup-test @@ -1,81 +1,81 @@ #!/bin/bash if ! which register-python-argcomplete3 &> /dev/null then echo "Missing register-python-argcomplete3 executable" exit 1 fi WTL_MIN_DOCKER_SERVER_VERSION=17.09.0 { WTL_DOCKER_SERVER_VERSION=$(docker version --format '{{.Server.Version}}') } &> /dev/null if [ $? -ne 0 ] then echo "Error running: docker" exit 1 fi if [[ "$WTL_DOCKER_SERVER_VERSION" != "$WTL_MIN_DOCKER_SERVER_VERSION" ]] && \ [[ "$(echo -e "$WTL_DOCKER_SERVER_VERSION\n$WTL_MIN_DOCKER_SERVER_VERSION" | sort -V | head -1)" != "$WTL_MIN_DOCKER_SERVER_VERSION" ]] then echo "Docker server $WTL_DOCKER_SERVER_VERSION < $WTL_MIN_DOCKER_SERVER_VERSION" exit 1 fi WTL_MIN_DOCKER_CLIENT_VERSION=17.09.0 { WTL_DOCKER_CLIENT_VERSION=$(docker version --format '{{.Client.Version}}') } &> /dev/null if [ $? -ne 0 ] then echo "Error running: docker" exit 1 fi if [[ "$WTL_DOCKER_CLIENT_VERSION" != "$WTL_MIN_DOCKER_CLIENT_VERSION" ]] && \ [[ "$(echo -e "$WTL_DOCKER_CLIENT_VERSION\n$WTL_MIN_DOCKER_CLIENT_VERSION" | sort -V | head -1)" != "$WTL_MIN_DOCKER_CLIENT_VERSION" ]] then echo "Docker client $WTL_DOCKER_CLIENT_VERSION < $WTL_MIN_DOCKER_CLIENT_VERSION" exit 1 fi WTL_MIN_DOCKER_COMPOSE_VERSION=1.19.0 { WTL_DOCKER_COMPOSE_VERSION=$(docker-compose version --short) } &> /dev/null if [ $? -ne 0 ] then echo "Error running: docker-compose" exit 1 fi if [[ "$WTL_DOCKER_COMPOSE_VERSION" != "$WTL_MIN_DOCKER_COMPOSE_VERSION" ]] && \ [[ "$(echo -e "$WTL_DOCKER_COMPOSE_VERSION\n$WTL_MIN_DOCKER_COMPOSE_VERSION" | sort -V | head -1)" != "$WTL_MIN_DOCKER_COMPOSE_VERSION" ]] then echo "docker-compose $WTL_DOCKER_COMPOSE_VERSION < $WTL_MIN_DOCKER_COMPOSE_VERSION" exit 1 fi WTL_MIN_GIT_VERSION=2.11.0 { WTL_GIT_VERSION=$(git --version | sed 's#git version ##g') } &> /dev/null if [ $? -ne 0 ] then echo "Error running: git" exit 1 fi if [[ "$WTL_GIT_VERSION" != "$WTL_MIN_GIT_VERSION" ]] && \ [[ "$(echo -e "$WTL_GIT_VERSION\n$WTL_MIN_GIT_VERSION" | sort -V | head -1)" != "$WTL_MIN_GIT_VERSION" ]] then echo "docker-compose $WTL_GIT_VERSION < $WTL_MIN_GIT_VERSION" exit 1 fi if test ! -f config/repositories.yml then - echo "Missing config/repositories.yml" - exit 1 + echo "Missing config/repositories.yml (not required)" +# exit 1 fi if test ! -f config/config.yml then echo "Missing config/config.yml" exit 1 fi echo "OK" diff --git a/config/repositories.example.yml b/config/repositories.example.yml index 3ef884c..57473a7 100644 --- a/config/repositories.example.yml +++ b/config/repositories.example.yml @@ -1,17 +1,17 @@ chapters-backend: - git: null + git: kde:scratch/atundo/wikitolearn-chapters-backend.git course-midtier: - git: null + git: kde:scratch/atundo/wikitolearn-course-midtier.git courses-backend: - git: null + git: kde:scratch/atundo/wikitolearn-courses-backend.git coursessecurity-backend: - git: null + git: kde:scratch/atundo/wikitolearn-coursessecurity-backend.git pages-backend: - git: null + git: kde:scratch/atundo/wikitolearn-pages-backend.git pwa-gateway: - git: null + git: kde:scratch/atundo/wikitolearn-pwa-gateway.git shared-services: - git: null + git: kde:scratch/atundo/wikitolearn-shared-services.git frontend: - git: null + git: kde:scratch/cristianbaldi/wikitolearn-frontend.git git_branch: migration