diff --git a/kdesrc-build b/kdesrc-build index 8cb9b33..a4cc72a 100755 --- a/kdesrc-build +++ b/kdesrc-build @@ -1,350 +1,350 @@ #!/usr/bin/env perl # Script to handle building KDE from source code. All of the configuration is # stored in the file ./kdesrc-buildrc (or ~/.kdesrc-buildrc, if that's not # present). # # Please also see the documentation that should be included with this program, # in the doc/ directory. # # Copyright © 2003 - 2018 Michael Pyne. # Home page: https://kdesrc-build.kde.org/ # # Copyright © 2005, 2006, 2008 - 2011 David Faure # Copyright © 2005 Thiago Macieira # Copyright © 2006 Stephan Kulow # Copyright © 2006, 2008 Dirk Mueller # ... and possibly others. Check the git source repository for specifics. # # This program is free software; you can redistribute it and/or modify it under # the terms of the GNU General Public License as published by the Free Software # Foundation; either version 2 of the License, or (at your option) any later # version. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more # details. # # You should have received a copy of the GNU General Public License along with # this program; if not, write to the Free Software Foundation, Inc., 51 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA # Adding an option? Grep for 'defaultGlobalOptions' in ksb::BuildContext --mpyne use FindBin qw($RealBin); use lib "$RealBin/../share/kdesrc-build/modules"; use lib "$RealBin/modules"; use strict; use warnings; use Carp; use Data::Dumper; use File::Find; # For our lndir reimplementation. use File::Path qw(remove_tree); use Mojo::IOLoop; use Mojo::Server::Daemon; use ksb::Debug; use ksb::Util; use ksb::Version qw(scriptVersion); use ksb::Application; use ksb::UserInterface::TTY; -use BackendServer; +use web::BackendServer; use 5.014; # Require Perl 5.14 # Make Perl 'plain die' exceptions use Carp::confess instead of their core # support. This is not supported by the Perl 5 authors but assuming it works # will be better than the alternative backtrace we get (which is to say, none) $SIG{__DIE__} = \&Carp::confess; $ksb::Version::SCRIPT_PATH = $RealBin; ### Script-global functions. # These functions might be called at runtime via log_command, using # log_command's support for symbolic execution of a named subroutine. Because # of that, they have been left in the top-level script. # # Everything else should be in an appropriate class. # Subroutine to recursively symlink a directory into another location, in a # similar fashion to how the XFree/X.org lndir() program does it. This is # reimplemented here since some systems lndir doesn't seem to work right. # # As a special exception to the GNU GPL, you may use and redistribute this # function however you would like (i.e. consider it public domain). # # The first parameter is the directory to symlink from. # The second parameter is the destination directory name. # # e.g. if you have $from/foo and $from/bar, lndir would create $to/foo and # $to/bar. # # All intervening directories will be created as needed. In addition, you # may safely run this function again if you only want to catch additional files # in the source directory. # # Note that this function will unconditionally output the files/directories # created, as it is meant to be a close match to lndir. # # RETURN VALUE: Boolean true (non-zero) if successful, Boolean false (0, "") # if unsuccessful. sub safe_lndir { my ($from, $to) = @_; # Create destination directory. if (not -e $to) { print "$to\n"; if (not pretending() and not super_mkdir($to)) { error ("Couldn't create directory r[$to]: b[r[$!]"); return 0; } } # Create closure callback subroutine. my $wanted = sub { my $dir = $File::Find::dir; my $file = $File::Find::fullname; $dir =~ s/$from/$to/; # Ignore the .svn directory and files. return if $dir =~ m,/\.svn,; # Create the directory. if (not -e $dir) { print "$dir\n"; if (not pretending()) { super_mkdir ($dir) or croak_runtime("Couldn't create directory $dir: $!"); } } # Symlink the file. Check if it's a regular file because File::Find # has no qualms about telling you you have a file called "foo/bar" # before pointing out that it was really a directory. if (-f $file and not -e "$dir/$_") { print "$dir/$_\n"; if (not pretending()) { symlink $File::Find::fullname, "$dir/$_" or croak_runtime("Couldn't create file $dir/$_: $!"); } } }; # Recursively descend from source dir using File::Find eval { find ({ 'wanted' => $wanted, 'follow_fast' => 1, 'follow_skip' => 2}, $from); }; if ($@) { error ("Unable to symlink $from to $to: $@"); return 0; } return 1; } # Subroutine to delete recursively, everything under the given directory, # unless we're in pretend mode. # # Used from ksb::BuildSystem to handle cleaning a build directory. # # i.e. the effect is similar to "rm -r $arg/* $arg/.*". # # This assumes we're called from a separate child process. Therefore the # normal logging routines are /not used/, since our output will be logged # by the parent kdesrc-build. # # The first parameter should be the absolute path to the directory to delete. # # Returns boolean true on success, boolean false on failure. sub prune_under_directory { my $dir = shift; my $errorRef; print "starting delete of $dir\n"; eval { remove_tree($dir, { keep_root => 1, error => \$errorRef }); }; if ($@ || @$errorRef) { error ("\tUnable to clean r[$dir]:\n\ty[b[$@]"); return 0; } return 1; } sub findMissingModules { # should be either strings of module names to be found or a listref containing # a list of modules where any one of which will work. my @requiredModules = ( 'HTTP::Tiny', 'IO::Socket::SSL', # Assume if Mojolicious::Lite is present, that whole framework is properly installed 'Mojolicious::Lite', 'Mojo::Promise', # That was a bad assumption 'Mojo::JSON', [qw(YAML::XS YAML::PP YAML::Syck)] ); my @missingModules; my $validateMod = sub { return eval "require $_[0]; 1;"; }; my $description; foreach my $neededModule (@requiredModules) { if (ref $neededModule) { # listref of options my @moduleOptions = @$neededModule; next if (ksb::Util::any (sub { $validateMod->($_); }, $neededModule)); $description = 'one of (' . join(', ', @moduleOptions) . ')'; } else { next if $validateMod->($neededModule); $description = $neededModule; } push @missingModules, $description; } return @missingModules; } # Rather than running an interactive build, launches a web server that can be # interacted with by and outside user interface, printing the URL to the server # on stdout and then remaining in the foreground until killed. sub launchBackend { # Manually setup the daemon so that we can figure out what port it # ends up on. my $daemon = Mojo::Server::Daemon->new( - app => BackendServer->new, + app => web::BackendServer->new, listen => ['http://localhost'], silent => 1, ); $daemon->start; # Grabs the socket to listen on my $port = $daemon->ports->[0] or do { say STDERR "Can't autodetect which TCP port was assigned!"; exit 1; }; say "http://localhost:$port"; Mojo::IOLoop->start unless Mojo::IOLoop->is_running; exit 0; } # Script starts. # Ensure some critical Perl modules are available so that the user isn't surprised # later with a Perl exception if(my @missingModuleDescriptions = findMissingModules()) { say <new(@ARGV); + my $app = web::BackendServer->new(@ARGV); my $ui = ksb::UserInterface::TTY->new($app); # Hack for debugging current state. if (exists $ENV{KDESRC_BUILD_DUMP_CONTEXT}) { local $Data::Dumper::Indent = 1; local $Data::Dumper::Sortkeys = 1; # This method call dumps the first list with the variables named by the # second list. print Data::Dumper->Dump([$app->ksb->context()], [qw(ctx)]); } push @atexit_subs, sub { $app->ksb->finish(99) }; # TODO: Reimplement --print-modules, --query modes, which wouldn't go through ->start my $result = $ui->start(); @atexit_subs = (); # Clear exit handlers # env driver is just the ~/.config/kde-env-*.sh, session driver is that + ~/.xsession my $ctx = $app->context; if ($ctx->getOption('install-environment-driver') || $ctx->getOption('install-session-driver')) { ksb::Application::_installCustomSessionDriver($ctx); } # Exits the script my $logdir = $app->context()->getLogDir(); note ("Your logs are saved in y[$logdir]"); exit $result; }; if (my $err = $@) { if (had_an_exception()) { say "kdesrc-build encountered an exceptional error condition:"; say " ========"; say " $err"; say " ========"; say "\tCan't continue, so stopping now."; say "\nPlease submit a bug against kdesrc-build on https://bugs.kde.org/" if ($err->{exception_type} eq 'Internal'); } else { # An exception was raised, but not one that kdesrc-build generated say "Encountered an error in the execution of the script."; say "The error reported was $err"; say "Please submit a bug against kdesrc-build on https://bugs.kde.org/"; } exit 99; } # vim: set et sw=4 ts=4: diff --git a/modules/ksb/UserInterface/TTY.pm b/modules/ksb/UserInterface/TTY.pm index 1845822..4f37696 100755 --- a/modules/ksb/UserInterface/TTY.pm +++ b/modules/ksb/UserInterface/TTY.pm @@ -1,230 +1,227 @@ #!/usr/bin/env perl package ksb::UserInterface::TTY 0.10; =pod =head1 NAME ksb::UserInterface::TTY -- A command-line interface to the kdesrc-build backend =head1 DESCRIPTION This class is used to show a user interface for a kdesrc-build run at the command line (as opposed to a browser-based or GUI interface). Since the kdesrc-build backend is now meant to be headless and controlled via a Web-style API set (powered by Mojolicious), this class manages the interaction with that backend, also using Mojolicious to power the HTTP and WebSocket requests necessary. =head1 SYNOPSIS - my $app = BackendServer->new(@ARGV); + my $app = web::BackendServer->new(@ARGV); my $ui = ksb::UserInterface::TTY->new($app); exit $ui->start(); # Blocks! Returns a shell-style return code =cut use strict; use warnings; use 5.014; use Mojo::Base -base; use Mojo::Server::Daemon; use Mojo::IOLoop; use Mojo::UserAgent; use Mojo::JSON qw(to_json); -# This is essentially ksb::Application but across a socket connection. It reads -# the options and module selectors like normal. -use BackendServer; use ksb::StatusView; use ksb::Util; use ksb::Debug; use IO::Handle; # For methods on event_stream file use List::Util qw(max); has ua => sub { Mojo::UserAgent->new->inactivity_timeout(0) }; has ui => sub { ksb::StatusView->new() }; has 'app'; sub new { my ($class, $app) = @_; my $self = $class->SUPER::new(app => $app); # Mojo::UserAgent can be tied to a Mojolicious application server directly to # handle relative URLs, which is perfect for what we want. Making this # attachment will startup the Web server behind the scenes and allow $ua to # make HTTP requests. $self->ua->server->app($app); $self->ua->server->app->log->level('fatal'); return $self; } sub _check_error { my $tx = shift; my $err = $tx->error or return; my $body = $tx->res->body // ''; open my $fh, '<', \$body; my ($first_line) = <$fh> // ''; $err->{message} .= "\n$first_line" if $first_line; die $err; }; # Just a giant huge promise handler that actually processes U/I events and # keeps the TTY up to date. Note the TTY-specific stuff is actually itself # buried in a separate class for now. sub start { my $self = shift; my $ui = $self->ui; my $ua = $self->ua; my $app = $self->app; my $result = 0; # notes errors from module builds or internal errors my @module_failures; # Open a file to log the event stream my $ctx = $app->context(); my $separator = ' '; my $dest = pretending() ? '/dev/null' : $ctx->getLogDirFor($ctx) . '/event-stream'; open my $event_stream, '>', $dest or croak_internal("Unable to open event log $!"); $event_stream->say("["); # Try to make it valid JSON syntax # This call just reads an option from the BuildContext as a sanity check $ua->get_p('/context/options/pretend')->then(sub { my $tx = shift; _check_error($tx); # If we get here things are mostly working? my $selectorsRef = $app->{selectors}; # We need to specifically ask for all modules if we're not passing a # specific list of modules to build. my $headers = { }; $headers->{'X-BuildAllModules'} = 1 unless @{$selectorsRef}; # Tell the backend which modules to build. return $ua->post_p('/modules', $headers, json => $selectorsRef); })->then(sub { my $tx = shift; _check_error($tx); # We've received a successful response from the backend that it's able to # build the requested modules, so proceed to setup the U/I and start the # build. return $ua->websocket_p('/events'); })->then(sub { # Websocket Event handler my $ws = shift; my $everFailed = 0; my $stop_promise = Mojo::Promise->new; # Websockets seem to be inherently event-driven instead of simply # client/server. So attach the event handlers and then return to the event # loop to await progress. $ws->on(json => sub { # This handler is called by the backend when there is something notable # to report my ($ws, $resultRef) = @_; foreach my $modRef (@{$resultRef}) { # Update the U/I eval { $ui->notifyEvent($modRef); $event_stream->say($separator . to_json($modRef)); $separator = ', '; }; if ($@) { $ws->finish; $stop_promise->reject($@); } # See ksb::StatusMonitor for where events defined if ($modRef->{event} eq 'phase_completed') { my $results = $modRef->{phase_completed}; push @module_failures, $results if $results->{result} eq 'error'; } if ($modRef->{event} eq 'build_done') { # We've reported the build is complete, activate the promise # holding things together. The value we pass is what is passed # to the next promise handler. $stop_promise->resolve(scalar @module_failures); } } }); $ws->on(finish => sub { # Shouldn't happen in a normal build but it's probably possible $stop_promise->reject; # ignored if we resolved first }); # Blocking call to kick off the build my $tx = $ua->post('/build'); if (my $err = $tx->error) { $stop_promise->reject('Unable to start build: ' . $err->{message}); } # Once we return here we'll wait in Mojolicious event loop for awhile until # the build is done, before moving into the promise handler below return $stop_promise; })->then(sub { # Build done, value comes from stop_promise->resolve above $result ||= shift; })->catch(sub { # Catches all errors in any of the prior promises my $err = shift; say "Error: ", $err->{code}, " ", $err->{message}; # See if we made it to an rc-file my $ctx = $app->ksb->context(); my $rcFile = $ctx ? $ctx->rcFile() // 'Unknown' : undef; say "Using configuration file found at $rcFile" if $rcFile; $result = 1; # error })->wait; $event_stream->say("]"); $event_stream->close() or $result = 1; _report_on_failures(@module_failures); say $result == 0 ? ":-)" : ":-("; return $result; }; sub _report_on_failures { my @failures = @_; my $max_width = max map { length ($_->{module}) } @failures; foreach my $mod (@failures) { my $module = $mod->{module}; my $phase = $mod->{phase}; my $log = $mod->{error_file}; my $padding = $max_width - length $module; $module .= (' ' x $padding); # Left-align $phase = 'setup buildsystem' if $phase eq 'buildsystem'; error("b[*] r[b[$module] failed to b[$phase]"); error("b[*]\tFind the log at file://$log") if $log; } } 1; diff --git a/modules/templates/layouts/default.html.ep b/modules/templates/layouts/default.html.ep deleted file mode 100644 index eb7fecf..0000000 --- a/modules/templates/layouts/default.html.ep +++ /dev/null @@ -1,31 +0,0 @@ - - - - -<%= title %> - - - - - -<%= content %> - diff --git a/modules/BackendServer.pm b/modules/web/BackendServer.pm similarity index 95% rename from modules/BackendServer.pm rename to modules/web/BackendServer.pm index 6b689f2..67acc54 100644 --- a/modules/BackendServer.pm +++ b/modules/web/BackendServer.pm @@ -1,251 +1,259 @@ -package BackendServer; +package web::BackendServer; # Make this subclass a Mojolicious app use Mojo::Base 'Mojolicious'; use Mojo::Util qw(trim); use ksb::Application; # This is written in a kind of domain-specific language for Mojolicious for # now, to setup a web server backend for clients / frontends to communicate # with. # See https://mojolicious.org/perldoc/Mojolicious/Guides/Tutorial has 'options'; has 'selectors'; sub new { my ($class, @opts) = @_; return $class->SUPER::new(options => [@opts]); } # Adds a helper method to each HTTP context object to return the # ksb::Application class in use sub make_new_ksb { my $c = shift; my $app = ksb::Application->new->setHeadless; my @selectors = $app->establishContext(@{$c->app->{options}}); $c->app->selectors([@selectors]); $c->app->log->info("Selectors are ", join(', ', @selectors)); return $app; } # Package-shared variables for helpers and closures my $LAST_RESULT; my $BUILD_PROMISE; my $IN_PROGRESS; my $KSB_APP; sub startup { my $self = shift; + # Force use of 'modules/web' as the home directory, would normally be + # 'modules' alone + $self->home($self->home->child('web')); + + # Fixup templates and public base directories + $self->static->paths->[0] = $self->home->child('public'); + $self->renderer->paths->[0] = $self->home->child('templates'); + $self->helper(ksb => sub { my ($c, $new_ksb) = @_; $KSB_APP //= make_new_ksb($c); $KSB_APP = $new_ksb if $new_ksb; return $KSB_APP; }); $self->helper(in_build => sub { $IN_PROGRESS }); $self->helper(context => sub { shift->ksb->context() }); my $r = $self->routes; $self->_generateRoutes; return; } sub _generateRoutes { my $self = shift; my $r = $self->routes; $r->get('/' => sub { my $c = shift; my $app = $c->ksb(); my $isApp = $app->isa('ksb::Application') ? 'app' : 'not app'; $c->stash(app => "Application is a $isApp"); $c->render(template => 'index'); } => 'index'); $r->post('/reset' => sub { my $c = shift; if ($c->in_build || !defined $LAST_RESULT) { $c->res->code(400); return $c->render; } my $old_result = $LAST_RESULT; $c->ksb(make_new_ksb($c)); undef $LAST_RESULT; $c->render(json => { last_result => $old_result }); }); $r->get('/context/options' => sub { my $c = shift; $c->render(json => $c->ksb->context()->{options}); }); $r->get('/context/options/:option' => sub { my $c = shift; my $ctx = $c->ksb->context(); my $opt = $c->param('option') or do { $c->res->code(400); return $c->render; }; if (defined $ctx->{options}->{$opt}) { $c->render(json => { $opt => $ctx->{options}->{$opt} }); } else { $c->res->code(404); $c->reply->not_found; } }); $r->get('/modules' => sub { my $c = shift; $c->render(json => $c->ksb->context()->moduleList()); } => 'module_lookup'); $r->get('/known_modules' => sub { my $c = shift; my $resolver = $c->ksb->{module_resolver}; my @setsAndModules = @{$resolver->{inputModulesAndOptions}}; my @output = map { $_->isa('ksb::ModuleSet') ? [ $_->name(), $_->moduleNamesToFind() ] : $_->name() # should be a ksb::Module } @setsAndModules; $c->render(json => \@output); }); $r->post('/modules' => sub { my $c = shift; my $selectorList = $c->req->json; my $build_all = $c->req->headers->header('X-BuildAllModules'); # Remove empty selectors my @modules = grep { !!$_ } map { trim($_ // '') } @{$selectorList}; # If not building all then ensure there's at least one module to build if ($c->in_build || !$selectorList || (!@modules && !$build_all) || (@modules && $build_all)) { $c->app->log->error("Something was wrong with modules to assign to build"); return $c->render(text => "Invalid request sent", status => 400); } eval { @modules = $c->ksb->modulesFromSelectors(@modules); $c->ksb->setModulesToProcess(@modules); }; if ($@) { return $c->render(text => $@->{message}, status => 400); } my $numSels = @modules; # count $c->render(json => ["$numSels handled"]); }, 'post_modules'); $r->get('/module/:modname' => sub { my $c = shift; my $name = $c->stash('modname'); my $module = $c->ksb->context()->lookupModule($name); if (!$module) { $c->render(template => 'does_not_exist'); return; } my $opts = { options => $module->{options}, persistent => $c->ksb->context()->{persistent_options}->{$name}, }; $c->render(json => $opts); }); $r->get('/module/:modname/logs/error' => sub { my $c = shift; my $name = $c->stash('modname'); $c->render(text => "TODO: Error logs for $name"); }); $r->get('/config' => sub { my $c = shift; $c->render(text => $c->ksb->context()->rcFile()); }); $r->post('/config' => sub { # TODO If new filename can be loaded, load it and reset application object die "Unimplemented"; }); $r->get('/build-metadata' => sub { die "Unimplemented"; }); $r->websocket('/events' => sub { my $c = shift; $c->inactivity_timeout(0); my $ctx = $c->ksb->context(); my $monitor = $ctx->statusMonitor(); # Send prior events the receiver wouldn't have received yet my @curEvents = $monitor->events(); $c->send({json => \@curEvents}); # Hook up an event handler to send future events as they're generated $monitor->on(newEvent => sub { my ($monitor, $resultRef) = @_; $c->on(drain => sub { $c->finish }) if ($resultRef->{event} eq 'build_done'); $c->send({json => [ $resultRef ]}); }); }); $r->get('/event_viewer' => sub { my $c = shift; $c->render(template => 'event_viewer'); }); $r->get('/building' => sub { my $c = shift; $c->render(text => $c->in_build ? 'True' : 'False'); }); $r->post('/build' => sub { my $c = shift; if ($c->in_build) { $c->res->code(400); $c->render(text => 'Build already in progress, cancel it first.'); return; } $c->app->log->debug('Starting build'); $IN_PROGRESS = 1; $BUILD_PROMISE = $c->ksb->startHeadlessBuild->finally(sub { my ($result) = @_; $c->app->log->debug("Build done"); $IN_PROGRESS = 0; return $LAST_RESULT = $result; }); $c->render(text => $c->url_for('event_viewer')->to_abs->to_string); }); } 1; diff --git a/modules/web/public/ksb-common.js b/modules/web/public/ksb-common.js new file mode 100644 index 0000000..ab212ae --- /dev/null +++ b/modules/web/public/ksb-common.js @@ -0,0 +1,2 @@ +// Common functions for use by all generated pages +const lkup = (name) => document.getElementById(name); diff --git a/modules/web/public/ksb-main.css b/modules/web/public/ksb-main.css new file mode 100644 index 0000000..d8245c2 --- /dev/null +++ b/modules/web/public/ksb-main.css @@ -0,0 +1,15 @@ +td.pending { + background-color: lightgray; +} + +td.done { + background-color: lightblue; +} + +td.done.success { + background-color: lightgreen; +} + +td.done.error { + background-color: pink; +} diff --git a/modules/templates/event_viewer.html.ep b/modules/web/templates/event_viewer.html.ep similarity index 100% rename from modules/templates/event_viewer.html.ep rename to modules/web/templates/event_viewer.html.ep diff --git a/modules/templates/index.html.ep b/modules/web/templates/index.html.ep similarity index 100% rename from modules/templates/index.html.ep rename to modules/web/templates/index.html.ep diff --git a/modules/web/templates/layouts/default.html.ep b/modules/web/templates/layouts/default.html.ep new file mode 100644 index 0000000..1fe2a0a --- /dev/null +++ b/modules/web/templates/layouts/default.html.ep @@ -0,0 +1,12 @@ + + + + + + + <%= title %> + + +<%= content %> +