diff --git a/tools/getcomponentrepolist.py b/tools/getcomponentrepolist.py new file mode 100755 index 0000000..372312d --- /dev/null +++ b/tools/getcomponentrepolist.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Friedrich W. H. Kossebau +# +# SPDX-License-Identifier: GPL-2.0-or-later + +import argparse +import requests +import sys + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('component', help='Name of component to list repos within') + + args = parser.parse_args() + + try: + r = requests.get('https://projects.kde.org/api/v1/find?active=true') + allprojects = r.json() + except Exception as exc: + sys.exit("Failed to get list of repos from projects.kde.org: {}".format(exc)) + + filtered_projects = [] + for project in allprojects: + # whitelist + if args.component and not project.startswith(args.component): + continue + + filtered_projects.append(project) + + print(" ".join(filtered_projects)) + +if __name__ == '__main__': + main() + diff --git a/tools/update-checkouts b/tools/update-checkouts index c1492f6..ef0356c 100755 --- a/tools/update-checkouts +++ b/tools/update-checkouts @@ -1,139 +1,175 @@ #!/bin/bash # Script to build up the source directories needed by the EBN and api.kde.org to be built # Requires kde-checkout-list.pl # Initial setup TOP=/srv/sources BINTOP=/home/api/bin #where kde-checkout-list.pl can be found # Functions git_manual_update() { if [ ! -d $1 ]; then git clone $2 $1 elif [ -d $1/.git ]; then cd $1 git reset --hard; git clean -f -d -x git pull else echo "*** $1 exists and is not a Git clone!" fi } #checkout_component: use kde-checkout-list to checkout the specified component and optional branch checkout_component() { cmd="$BINTOP/kde-checkout-list.pl --clone --prune --component=$1" if [ -z "$2" ]; then $cmd else $cmd --branch="$2" fi } +checkout_component_repos() +{ + component=$1 + REPOS=`$BINTOP/getcomponentrepolist.py $component` + success=$? + if [ $success -ne 0 ]; then + echo "*** Getting list of repos for $component failed, skipping update!" + return + fi + + # get & update repos + for repo in $REPOS; do + if [ ! -d $repo ]; then + git clone https://invent.kde.org/$repo.git $repo + elif [ -d $repo/.git ]; then + pushd $repo > /dev/null + git reset --hard; + git clean -f -d -x + git pull + popd > /dev/null + else + echo "*** $repo exists and is not a Git clone!" + fi + done + + # prune no longer existing ones + OLDREPOS=$component/*/.git + for oldrepo in $OLDREPOS; do + oldrepo=${oldrepo%"/.git"} + if [[ ! $REPOS =~ (^|[[:space:]])$oldrepo($|[[:space:]]) ]]; then + echo "*** Pruning outdated repository $oldrepo" + rm -rf $oldrepo + fi + done +} + if [ ! -x $BINTOP/kde-checkout-list.pl ]; then echo "Cannot locate the kde-checkout-list.pl program in $BINTOP. Exiting..." exit 1 fi # Lets go! # Switch to the top... mkdir -p $TOP # Update KDE sources... cd $TOP checkout_component "kde" ( cd $TOP; rm -f kde-4.x; ln -sf kde kde-4.x ) # Frameworks mkdir -p $TOP/frameworks cd $TOP/frameworks -checkout_component "frameworks" +checkout_component_repos "frameworks" ( cd $TOP/frameworks; rm -f frameworks5; ln -s frameworks frameworks5 ) # Update Extragear.... cd $TOP checkout_component "extragear" # Update Playground... cd $TOP checkout_component "playground" # Update KDE Support cd $TOP checkout_component "kdesupport" # Update KDE Review cd $TOP checkout_component "kdereview" # Pull in Calligra... mkdir -p $TOP/calligra cd $TOP/calligra checkout_component "calligra" ( cd $TOP; rm -f bundled-apps; ln -s calligra bundled-apps ) # Put in place the quality checkout symlink # We already have a clone of this in $HOME (per update-scripts) so just reuse that one.... ( cd $TOP; rm -f quality; ln -sf $HOME/quality-kde-org quality ) # Update the website rsync -av --del $TOP/websites/quality-kde-org/website/api.kde.org/ /srv/www/api.kde.org/ \ --exclude doc \ --exclude 11-api \ --exclude 11-api \ --exclude 2.0-api \ --exclude 2.1-api \ --exclude 2.2-api \ --exclude 3.0-api \ --exclude 3.1-api \ --exclude 3.5-api \ --exclude 4.0-api \ --exclude 4.10-api \ --exclude 4.11-api \ --exclude 4.12-api \ --exclude 4.13-api \ --exclude 4.14-api \ --exclude 4.1-api \ --exclude 4.2-api \ --exclude 4.3-api \ --exclude 4.4-api \ --exclude 4.5-api \ --exclude 4.6-api \ --exclude 4.7-api \ --exclude 4.8-api \ --exclude 4.9-api \ --exclude 4.x-api \ --exclude apidocs \ --exclude bundled-apps-api \ --exclude cmake \ --exclude common \ --exclude data \ --exclude dependency-information.tar \ --exclude ecm \ --exclude extragear-api \ --exclude frameworks-api \ --exclude kdereview-api \ --exclude kdesupport-api \ --exclude man \ --exclude playground-api \ --exclude pykde-4.1-api \ --exclude pykde-4.2-api \ --exclude pykde-4.3-api \ --exclude pykde-4.4-api \ --exclude pykde-4.5-api \ --exclude pykde-4.7-api \ --exclude qch \ --exclude qyoto-4.0.5-api \ --exclude qyoto-4.0.6-api \ --exclude qyoto-api \ --exclude robots.txt \ --exclude search \ --exclude searchmaps \ --exclude stable \ --exclude unstable # We need Qt too cd $TOP git_manual_update "kde-qt" "git://code.qt.io/qt/qt.git"