diff --git a/kdesrc-build b/kdesrc-build index e2b79eb..ab3f99a 100755 --- a/kdesrc-build +++ b/kdesrc-build @@ -1,396 +1,408 @@ #!/usr/bin/env perl # Script to handle building KDE from source code. All of the configuration is # stored in the file ./kdesrc-buildrc (or ~/.kdesrc-buildrc, if that's not # present). # # Please also see the documentation that should be included with this program, # in the doc/ directory. # # Copyright © 2003 - 2018 Michael Pyne. # Home page: https://kdesrc-build.kde.org/ # # Copyright © 2005, 2006, 2008 - 2011 David Faure # Copyright © 2005 Thiago Macieira # Copyright © 2006 Stephan Kulow # Copyright © 2006, 2008 Dirk Mueller # ... and possibly others. Check the git source repository for specifics. # # This program is free software; you can redistribute it and/or modify it under # the terms of the GNU General Public License as published by the Free Software # Foundation; either version 2 of the License, or (at your option) any later # version. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more # details. # # You should have received a copy of the GNU General Public License along with # this program; if not, write to the Free Software Foundation, Inc., 51 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA # Adding an option? Grep for 'defaultGlobalOptions' in ksb::BuildContext --mpyne use 5.014; # Require Perl 5.14 use strict; use warnings; -# On many container-based distros, even FindBin is missing to conserve space. -# But we can use File::Spec to do nearly the same. -my $RealBin; +# "our" makes this a package-shared variable accessible outside of this package +# by referring to $main::baseDataDir. +our $baseDataDir; # where to find support files like modules and dependencies +my $baseBinDir; # where to find kdesrc-build itself this run my $modPath; -# The File::Spec calls have to run when parsing (i.e. in BEGIN) to make the -# 'use lib' below work (which itself implicitly uses BEGIN { }) +# Find our base directory to find modules and data under. On many +# container-based distros, even FindBin is missing to conserve space. But we +# can use File::Spec to do nearly the same. +# These calls have to run when parsing (i.e. in BEGIN) to make the 'use lib' +# below work (which itself implicitly uses BEGIN { }) BEGIN { use File::Spec; # resolve symlinks my $scriptPath = $0; for (1..16) { last unless -l $scriptPath; $scriptPath = readlink $scriptPath; } - die "Too many symlinks followed looking for script" if -l $scriptPath; + die "Too many symlinks followed looking for script" + if -l $scriptPath; my ($volume, $directories, $script) = File::Spec->splitpath($scriptPath); + my $baseDir = File::Spec->catpath($volume, $directories, ''); - $RealBin = File::Spec->catpath($volume, $directories, ''); - die "Couldn't find base directory!" unless $RealBin; + die "Couldn't find base directory!" + unless $baseDir; + + $baseBinDir = File::Spec->rel2abs('.', $baseDir); # Use modules in git repo if running from git dir, otherwise assume - # system install - $modPath = File::Spec->rel2abs('modules', $RealBin); - $modPath = ($RealBin =~ s,/bin/?$,/share/kdesrc-build/modules,r) - unless -d $modPath; + # system install. + if (-d "$baseDir/modules" && -e "$baseDir/kdesrc-build") { + $baseDataDir = $baseBinDir; + } elsif (-d "$baseDir/../share/kdesrc-build/modules") { + $baseDataDir = File::Spec->rel2abs("../share/kdesrc-build", $baseDir); + } - die "Couldn't find modules for kdesrc-build!" unless $modPath; + $modPath = "$baseDataDir/modules"; + die "Couldn't find modules for kdesrc-build from $baseDir!" + unless -d $modPath; } use lib "$modPath"; # Make ksb:: modules available sub dumpError { my $err = $@; open my $fh, '>>', "error-$$.log" or return; my $time = localtime; say $fh $time; say $fh $@; } # When running in a limited environment, we might not be able to load # our modules although we can find them. In this case we should help user # by setting up system dependencies. eval { if (grep { $_ eq '--initial-setup' } @ARGV) { require ksb::FirstRun; require ksb::Debug; ksb::Debug::setColorfulOutput(1); - exit ksb::FirstRun::setupUserSystem(File::Spec->rel2abs($RealBin)); + exit ksb::FirstRun::setupUserSystem($baseDataDir); } }; if ($@) { dumpError(); say STDERR <import(); ksb::Util->import(); ksb::BuildException->import(); ksb::Version->import(qw(scriptVersion)); ksb::Application->import(); # Make Perl 'plain die' exceptions use Carp::confess instead of their core # support. This is not supported by the Perl 5 authors but assuming it works # will be better than the alternative backtrace we get (which is to say, none) $SIG{__DIE__} = \&Carp::confess; -ksb::Version->path($RealBin); +ksb::Version->path($baseBinDir); ### Script-global functions. # These functions might be called at runtime via log_command, using # log_command's support for symbolic execution of a named subroutine. Because # of that, they have been left in the top-level script. # # Everything else should be in an appropriate class. # Subroutine to recursively symlink a directory into another location, in a # similar fashion to how the XFree/X.org lndir() program does it. This is # reimplemented here since some systems lndir doesn't seem to work right. # # Used from ksb::l10nSystem # # As a special exception to the GNU GPL, you may use and redistribute this # function however you would like (i.e. consider it public domain). # # The first parameter is the directory to symlink from. # The second parameter is the destination directory name. # # e.g. if you have $from/foo and $from/bar, lndir would create $to/foo and # $to/bar. # # All intervening directories will be created as needed. In addition, you # may safely run this function again if you only want to catch additional files # in the source directory. # # Note that this function will unconditionally output the files/directories # created, as it is meant to be a close match to lndir. # # RETURN VALUE: Boolean true (non-zero) if successful, Boolean false (0, "") # if unsuccessful. sub safe_lndir { my ($from, $to) = @_; # Create destination directory. if (not -e $to) { print "$to\n"; if (not pretending() and not super_mkdir($to)) { error ("Couldn't create directory r[$to]: b[r[$!]"); return 0; } } # Create closure callback subroutine. my $wanted = sub { my $dir = $File::Find::dir; my $file = $File::Find::fullname; $dir =~ s/$from/$to/; # Ignore the .svn directory and files. return if $dir =~ m,/\.svn,; # Create the directory. if (not -e $dir) { print "$dir\n"; if (not pretending()) { super_mkdir ($dir) or croak_runtime("Couldn't create directory $dir: $!"); } } # Symlink the file. Check if it's a regular file because File::Find # has no qualms about telling you you have a file called "foo/bar" # before pointing out that it was really a directory. if (-f $file and not -e "$dir/$_") { print "$dir/$_\n"; if (not pretending()) { symlink $File::Find::fullname, "$dir/$_" or croak_runtime("Couldn't create file $dir/$_: $!"); } } }; # Recursively descend from source dir using File::Find eval { find ({ 'wanted' => $wanted, 'follow_fast' => 1, 'follow_skip' => 2}, $from); }; if ($@) { error ("Unable to symlink $from to $to: $@"); return 0; } return 1; } # Subroutine to delete recursively, everything under the given directory, # unless we're in pretend mode. # # Used from ksb::BuildSystem to handle cleaning a build directory. # # i.e. the effect is similar to "rm -r $arg/* $arg/.*". # # This assumes we're called from a separate child process. Therefore the # normal logging routines are /not used/, since our output will be logged # by the parent kdesrc-build. # # The first parameter should be the absolute path to the directory to delete. # # Returns boolean true on success, boolean false on failure. sub prune_under_directory { my $dir = shift; my $errorRef; print "starting delete of $dir\n"; eval { remove_tree($dir, { keep_root => 1, error => \$errorRef }); }; if ($@ || @$errorRef) { error ("\tUnable to clean r[$dir]:\n\ty[b[$@]"); return 0; } return 1; } sub findMissingModules { # should be either strings of module names to be found or a listref containing # a list of modules where any one of which will work. my @requiredModules = ( 'HTTP::Tiny', 'IO::Socket::SSL', [qw(JSON::XS JSON::PP)], [qw(YAML::XS YAML::PP YAML::Syck)] ); my @missingModules; my $validateMod = sub { return eval "require $_[0]; 1;"; }; my $description; foreach my $neededModule (@requiredModules) { if (ref $neededModule) { # listref of options my @moduleOptions = @$neededModule; next if (ksb::Util::any (sub { $validateMod->($_); }, $neededModule)); $description = 'one of (' . join(', ', @moduleOptions) . ')'; } else { next if $validateMod->($neededModule); $description = $neededModule; } push @missingModules, $description; } return @missingModules; } # Script starts. # Ensure some critical Perl modules are available so that the user isn't surprised # later with a Perl exception if(my @missingModuleDescriptions = findMissingModules()) { say <new(@ARGV); push @atexit_subs, sub { $app->finish(99) }; my $result = $app->runAllModulePhases(); @atexit_subs = (); # Clear exit handlers $app->finish($result); }; if (my $err = $@) { if (had_an_exception()) { print "kdesrc-build encountered an exceptional error condition:\n"; print " ========\n"; print " $err\n"; print " ========\n"; print "\tCan't continue, so stopping now.\n"; if ($err->{'exception_type'} eq 'Internal') { print "\nPlease submit a bug against kdesrc-build on https://bugs.kde.org/\n" } } else { # We encountered an error. print "Encountered an error in the execution of the script.\n"; print "The error reported was $err\n"; print "Please submit a bug against kdesrc-build on https://bugs.kde.org/\n"; } exit 99; } # vim: set et sw=4 ts=4 fdm=marker: diff --git a/modules/ksb/Application.pm b/modules/ksb/Application.pm index 2a40fd9..0513c52 100644 --- a/modules/ksb/Application.pm +++ b/modules/ksb/Application.pm @@ -1,2623 +1,2609 @@ package ksb::Application 0.20; # Class: Application # # Contains the application-layer logic (i.e. creating a build context, reading # options, parsing command-line, etc.) use strict; use warnings; use 5.014; no if $] >= 5.018, 'warnings', 'experimental::smartmatch'; use ksb::Debug; -use ksb::Util; +use ksb::Util 0.35; use ksb::BuildContext 0.35; use ksb::BuildSystem::QMake; use ksb::BuildException 0.20; use ksb::FirstRun; use ksb::Module; use ksb::ModuleResolver 0.20; use ksb::ModuleSet 0.20; use ksb::ModuleSet::KDEProjects; use ksb::OSSupport; use ksb::RecursiveFH; use ksb::DependencyResolver 0.20; use ksb::IPC::Pipe 0.20; use ksb::IPC::Null; use ksb::Updater::Git; use ksb::Version qw(scriptVersion); use List::Util qw(first min); use File::Basename; # basename, dirname use File::Glob ':glob'; use POSIX qw(:sys_wait_h _exit :errno_h); use Getopt::Long qw(GetOptionsFromArray :config gnu_getopt nobundling); use IO::Handle; use IO::Select; ### Package-specific variables (not shared outside this file). use constant { # We use a named remote to make some git commands work that don't accept the # full path. KDE_PROJECT_ID => 'kde-projects', # git-repository-base for kde_projects.xml }; ### Package methods sub new { my ($class, @options) = @_; my $self = bless { context => ksb::BuildContext->new(), metadata_module => undef, run_mode => 'build', modules => undef, module_factory => undef, # ref to sub that makes a new Module. # See generateModuleList _base_pid => $$, # See finish() }, $class; # Default to colorized output if sending to TTY ksb::Debug::setColorfulOutput(-t STDOUT); my @moduleList = $self->generateModuleList(@options); $self->{modules} = \@moduleList; if (!@moduleList) { print "No modules to build, exiting.\n"; exit 0; } $self->context()->setupOperatingEnvironment(); # i.e. niceness, ulimits, etc. # After this call, we must run the finish() method # to cleanly complete process execution. if (!pretending() && !$self->context()->takeLock()) { print "$0 is already running!\n"; exit 1; # Don't finish(), it's not our lockfile!! } # Install signal handlers to ensure that the lockfile gets closed. _installSignalHandlers(sub { note ("Signal received, terminating."); @main::atexit_subs = (); # Remove their finish, doin' it manually $self->finish(5); }); return $self; } # Method: _readCommandLineOptionsAndSelectors # # Returns a list of module/module-set selectors, selected module/module-set # options, and global options, based on the command-line arguments passed to # this function. # # This is a package method, should be called as # $app->_readCommandLineOptionsAndSelectors # # Phase: # initialization - Do not call from this function. # # Parameters: # cmdlineOptions - hashref to hold parsed modules options to be applied later. # *Note* this must be done separately, it is not handled by this subroutine. # Global options will be stored in a hashref at $cmdlineOptions->{global}. # Module or module-set options will be stored in a hashref at # $cmdlineOptions->{$moduleName} (it will be necessary to disambiguate # later in the run whether it is a module set or a single module). # # If the global option 'start-program' is set, then the program to start and # its options will be found in a listref pointed to under the # 'start-program' option. # # selectors - listref to hold the list of module or module-set selectors to # build, in the order desired by the user. These will just be strings, the # caller will have to figure out whether the selector is a module or # module-set, and create any needed objects, and then set the recommended # options as listed in cmdlineOptions. # # ctx - to hold the global build state. # # @options - The remainder of the arguments are treated as command line # arguments to process. # # Returns: # Nothing. An exception will be raised on failure, or this function may quit # the program directly (e.g. to handle --help, --usage). sub _readCommandLineOptionsAndSelectors { my $self = shift; my ($cmdlineOptionsRef, $selectorsRef, $ctx, @options) = @_; my $phases = $ctx->phases(); my @savedOptions = @options; # Copied for use in debugging. my $os = ksb::OSSupport->new; my $version = "kdesrc-build " . scriptVersion(); my $author = < Many people have contributed code, bugfixes, and documentation. Please report bugs using the KDE Bugzilla, at https://bugs.kde.org/ DONE # Getopt::Long will store options in %foundOptions, since that is what we # pass in. To allow for custom subroutines to handle an option it is # required that the sub *also* be in %foundOptions... whereupon it will # promptly be overwritten if we're not careful. Instead we let the custom # subs save to %auxOptions, and read those in back over it later. my (%foundOptions, %auxOptions); %foundOptions = ( 'show-info' => sub { say $version; say "OS: ", $os->vendorID(); exit }, version => sub { say $version; exit }, author => sub { say $author; exit }, help => sub { _showHelpMessage(); exit 0 }, install => sub { $self->{run_mode} = 'install'; $phases->phases('install'); }, uninstall => sub { $self->{run_mode} = 'uninstall'; $phases->phases('uninstall'); }, 'no-src' => sub { $phases->filterOutPhase('update'); }, 'no-install' => sub { $phases->filterOutPhase('install'); }, 'no-snapshots' => sub { # The documented form of disable-snapshots $auxOptions{'disable-snapshots'} = 1; }, 'no-tests' => sub { # The "right thing" to do $phases->filterOutPhase('test'); # What actually works at this point. $foundOptions{'run-tests'} = 0; }, 'no-build' => sub { $phases->filterOutPhase('build'); }, # Mostly equivalent to the above 'src-only' => sub { $phases->phases('update'); # We have an auto-switching function that we only want to run # if --src-only was passed to the command line, so we still # need to set a flag for it. $foundOptions{'allow-auto-repo-move'} = 1; }, 'build-only' => sub { $phases->phases('build'); }, 'install-only' => sub { $self->{run_mode} = 'install'; $phases->phases('install'); }, prefix => sub { my ($optName, $arg) = @_; $auxOptions{prefix} = $arg; $foundOptions{kdedir} = $arg; #TODO: Still needed for compat? $foundOptions{reconfigure} = 1; }, query => sub { my (undef, $arg) = @_; my $validMode = qr/^[a-zA-Z0-9_][a-zA-Z0-9_-]*$/; die("Invalid query mode $arg") unless $arg =~ $validMode; # Add useful aliases $arg = 'source-dir' if $arg =~ /^src-?dir$/; $arg = 'build-dir' if $arg =~ /^build-?dir$/; $arg = 'install-dir' if $arg eq 'prefix'; $self->{run_mode} = 'query'; $auxOptions{query} = $arg; $auxOptions{pretend} = 1; # Implied pretend mode }, pretend => sub { # Set pretend mode but also force the build process to run. $auxOptions{pretend} = 1; $foundOptions{'build-when-unchanged'} = 1; }, resume => sub { $auxOptions{resume} = 1; $phases->filterOutPhase('update'); # Implied --no-src $foundOptions{'no-metadata'} = 1; # Implied --no-metadata }, verbose => sub { $foundOptions{'debug-level'} = ksb::Debug::WHISPER }, quiet => sub { $foundOptions{'debug-level'} = ksb::Debug::NOTE }, 'really-quiet' => sub { $foundOptions{'debug-level'} = ksb::Debug::WARNING }, debug => sub { $foundOptions{'debug-level'} = ksb::Debug::DEBUG; debug ("Commandline was: ", join(', ', @savedOptions)); }, # Hack to set module options 'set-module-option-value' => sub { my ($optName, $arg) = @_; my ($module, $option, $value) = split (',', $arg, 3); if ($module && $option) { $cmdlineOptionsRef->{$module} //= { }; $cmdlineOptionsRef->{$module}->{$option} = $value; } }, # Getopt::Long doesn't set these up for us even though we specify an # array. Set them up ourselves. 'start-program' => [ ], 'ignore-modules' => [ ], # Module selectors, the <> is Getopt::Long shortcut for an # unrecognized non-option value (i.e. an actual argument) '<>' => sub { my $arg = shift; push @{$selectorsRef}, $arg; }, ); # Handle any "cmdline-eligible" options not already covered. my $flagHandler = sub { my ($optName, $optValue) = @_; # Assume to set if nothing provided. $optValue = 1 if (!defined $optValue or $optValue eq ''); $optValue = 0 if lc($optValue) eq 'false'; $optValue = 0 if !$optValue; $auxOptions{$optName} = $optValue; }; foreach my $option (keys %ksb::BuildContext::defaultGlobalFlags) { if (!exists $foundOptions{$option}) { $foundOptions{$option} = $flagHandler; # A ref to a sub here! } } # Actually read the options. my $optsSuccess = GetOptionsFromArray(\@options, \%foundOptions, # Options here should not duplicate the flags and options defined below # from ksb::BuildContext! 'version|v', 'author', 'help', 'show-info', 'install', 'uninstall', 'no-src|no-svn', 'no-install', 'no-build', 'no-tests', 'build-when-unchanged|force-build', 'no-metadata', 'verbose', 'quiet|quite|q', 'really-quiet', 'debug', 'reconfigure', 'colorful-output|color!', 'async!', 'src-only|svn-only', 'build-only', 'install-only', 'build-system-only', 'rc-file=s', 'prefix=s', 'niceness|nice:10', 'ignore-modules=s{,}', 'print-modules', 'pretend|dry-run|p', 'refresh-build', 'query=s', 'start-program|run=s{,}', 'revision=i', 'resume-from=s', 'resume-after=s', 'rebuild-failures', 'resume', 'stop-after=s', 'stop-before=s', 'set-module-option-value=s', 'metadata-only', 'include-dependencies', # Special sub used (see above), but have to tell Getopt::Long to look # for negatable boolean flags (map { "$_!" } (keys %ksb::BuildContext::defaultGlobalFlags)), # Default handling fine, still have to ask for strings. (map { "$_:s" } (keys %ksb::BuildContext::defaultGlobalOptions)), '<>', # Required to read non-option args ); if (!$optsSuccess) { croak_runtime("Error reading command-line options."); } # To store the values we found, need to strip out the values that are # subroutines, as those are the ones we created. Alternately, place the # subs inline as an argument to the appropriate option in the # GetOptionsFromArray call above, but that's ugly too. my @readOptionNames = grep { ref($foundOptions{$_}) ne 'CODE' } (keys %foundOptions); # Slice assignment: $left{$key} = $right{$key} foreach $key (@keys), but # with hashref syntax everywhere. @{ $cmdlineOptionsRef->{'global'} }{@readOptionNames} = @foundOptions{@readOptionNames}; @{ $cmdlineOptionsRef->{'global'} }{keys %auxOptions} = values %auxOptions; } # Generates the build context and module list based on the command line options # and module selectors provided, resolves dependencies on those modules if needed, # filters out ignored or skipped modules, and sets up the module factory. # # After this function is called all module set selectors will have been # expanded, and we will have downloaded kde-projects metadata. # # Returns: List of Modules to build. sub generateModuleList { my $self = shift; my @argv = @_; # Note: Don't change the order around unless you're sure of what you're # doing. my $ctx = $self->context(); my $cmdlineOptions = { global => { }, }; my $cmdlineGlobalOptions = $cmdlineOptions->{global}; my $deferredOptions = { }; # 'options' blocks # Process --help, --install, etc. first. my @selectors; $self->_readCommandLineOptionsAndSelectors($cmdlineOptions, \@selectors, $ctx, @argv); # Convert list to hash for lookup my %ignoredSelectors = map { $_, 1 } @{$cmdlineGlobalOptions->{'ignore-modules'}}; my @startProgramAndArgs = @{$cmdlineGlobalOptions->{'start-program'}}; delete @{$cmdlineGlobalOptions}{qw/ignore-modules start-program/}; # rc-file needs special handling. if (exists $cmdlineGlobalOptions->{'rc-file'} && $cmdlineGlobalOptions->{'rc-file'}) { $ctx->setRcFile($cmdlineGlobalOptions->{'rc-file'}); } # disable async if only running a single phase. $cmdlineGlobalOptions->{async} = 0 if (scalar $ctx->phases()->phases() == 1); my $fh = $ctx->loadRcFile(); $ctx->loadPersistentOptions(); if (exists $cmdlineGlobalOptions->{'resume'}) { my $moduleList = $ctx->getPersistentOption('global', 'resume-list'); if (!$moduleList) { error ("b[--resume] specified, but unable to find resume point!"); error ("Perhaps try b[--resume-from] or b[--resume-after]?"); croak_runtime("Invalid --resume flag"); } unshift @selectors, split(/,\s*/, $moduleList); } if (exists $cmdlineGlobalOptions->{'rebuild-failures'}) { my $moduleList = $ctx->getPersistentOption('global', 'last-failed-module-list'); if (!$moduleList) { error ("b[y[--rebuild-failures] was specified, but unable to determine"); error ("which modules have previously failed to build."); croak_runtime("Invalid --rebuild-failures flag"); } unshift @selectors, split(/,\s*/, $moduleList); } # _readConfigurationOptions will add pending global opts to ctx while ensuring # returned modules/sets have any such options stripped out. It will also add # module-specific options to any returned modules/sets. my @optionModulesAndSets = _readConfigurationOptions($ctx, $fh, $deferredOptions); close $fh; # Check if we're supposed to drop into an interactive shell instead. If so, # here's the stop off point. if (@startProgramAndArgs) { $ctx->setupEnvironment(); # Read options from set-env $ctx->commitEnvironmentChanges(); # Apply env options to environment _executeCommandLineProgram(@startProgramAndArgs); # noreturn } # Everything else in cmdlineOptions should be OK to apply directly as a module # or context option. $ctx->setOption(%{$cmdlineGlobalOptions}); # Selecting modules or module sets would requires having the KDE # build metadata (kde-build-metadata and sysadmin/repo-metadata) # available. $ctx->setKDEDependenciesMetadataModuleNeeded(); $ctx->setKDEProjectsMetadataModuleNeeded(); if (!exists $ENV{HARNESS_ACTIVE}) { # Running in a test harness, avoid downloading metadata which will be # ignored in the test or making changes to git config ksb::Updater::Git::verifyGitConfig(); $self->_downloadKDEProjectMetadata(); } # The user might only want metadata to update to allow for a later # --pretend run, check for that here. if (exists $cmdlineGlobalOptions->{'metadata-only'}) { return; } # At this point we have our list of candidate modules / module-sets (as read in # from rc-file). The module sets have not been expanded into modules. # We also might have cmdline "selectors" to determine which modules or # module-sets to choose. First let's select module sets, and expand them. my @globalCmdlineArgs = keys %{$cmdlineGlobalOptions}; my $commandLineModules = scalar @selectors; my $moduleResolver = ksb::ModuleResolver->new($ctx); $moduleResolver->setCmdlineOptions($cmdlineOptions); $moduleResolver->setDeferredOptions($deferredOptions); $moduleResolver->setInputModulesAndOptions(\@optionModulesAndSets); $moduleResolver->setIgnoredSelectors([keys %ignoredSelectors]); $self->_defineNewModuleFactory($moduleResolver); my @modules; if ($commandLineModules) { @modules = $moduleResolver->resolveSelectorsIntoModules(@selectors); } else { # Build everything in the rc-file, in the order specified. @modules = $moduleResolver->expandModuleSets(@optionModulesAndSets); if ($ctx->getOption('kde-languages')) { @modules = _expandl10nModules($ctx, @modules); } } # If modules were on the command line then they are effectively forced to # process unless overridden by command line options as well. If phases # *were* overridden on the command line, then no update pass is required # (all modules already have correct phases) @modules = _updateModulePhases(@modules) unless $commandLineModules; # TODO: Verify this does anything still my $metadataModule = $ctx->getKDEDependenciesMetadataModule(); $ctx->addToIgnoreList($metadataModule->scm()->ignoredModules()); # Remove modules that are explicitly blanked out in their branch-group # i.e. those modules where they *have* a branch-group, and it's set to # be empty (""). my $resolver = $ctx->moduleBranchGroupResolver(); my $branchGroup = $ctx->effectiveBranchGroup(); @modules = grep { my $branch = $_->isKDEProject() ? $resolver->findModuleBranch($_->fullProjectPath(), $branchGroup) : 1; # Just a placeholder truthy value whisper ("Removing ", $_->fullProjectPath(), " due to branch-group") if (defined $branch and !$branch); (!defined $branch or $branch); # This is the actual test } (@modules); @modules = $self->_resolveModuleDependencies(@modules); # Filter --resume-foo options. This might be a second pass, but that should # be OK since there's nothing different going on from the first pass (in # resolveSelectorsIntoModules) in that event. @modules = _applyModuleFilters($ctx, @modules); # Check for ignored modules (post-expansion) @modules = grep { ! exists $ignoredSelectors{$_->name()} } @modules; return @modules; } # Causes kde-projects metadata to be downloaded (unless --pretend, --no-src, or # --no-metadata is in effect, although we'll download even in --pretend if # nothing is available). # # No return value. sub _downloadKDEProjectMetadata { my $self = shift; my $ctx = $self->context(); my $updateStillNeeded = 0; my $wasPretending = pretending(); eval { for my $metadataModule ( $ctx->getKDEDependenciesMetadataModule(), $ctx->getKDEProjectsMetadataModule()) { my $sourceDir = $metadataModule->getSourceDir(); super_mkdir($sourceDir); my $moduleSource = $metadataModule->fullpath('source'); my $updateDesired = !$ctx->getOption('no-metadata') && $ctx->phases()->has('update'); my $updateNeeded = (! -e $moduleSource) || is_dir_empty($moduleSource); my $lastUpdate = $ctx->getPersistentOption('global', 'last-metadata-update') // 0; $updateStillNeeded ||= $updateNeeded; if (!$updateDesired && $updateNeeded && (time - ($lastUpdate)) >= 7200) { warning (" r[b[*] Skipping build metadata update, but it hasn't been updated recently!"); } if ($updateNeeded && pretending()) { warning (" y[b[*] Ignoring y[b[--pretend] option to download required metadata\n" . " y[b[*] --pretend mode will resume after metadata is available."); ksb::Debug::setPretending(0); } if ($updateDesired && (!pretending() || $updateNeeded)) { $metadataModule->scm()->updateInternal(); $ctx->setPersistentOption('global', 'last-metadata-update', time); } ksb::Debug::setPretending($wasPretending); } }; my $err = $@; ksb::Debug::setPretending($wasPretending); if ($err) { die $err if $updateStillNeeded; # Assume previously-updated metadata will work if not updating warning (" b[r[*] Unable to download required metadata for build process"); warning (" b[r[*] Will attempt to press onward..."); warning (" b[r[*] Exception message: $@"); } } # Returns a list of Modules in the proper build order according to the # kde-build-metadata dependency information. # # The kde-build-metadata repository must have already been updated, and the # module factory must be setup. The Modules to reorder must be passed as # arguments. sub _resolveModuleDependencies { my $self = shift; my $ctx = $self->context(); my $metadataModule = $ctx->getKDEDependenciesMetadataModule(); my @modules = @_; @modules = eval { my $dependencyResolver = ksb::DependencyResolver->new($self->{module_factory}); my $branchGroup = $ctx->effectiveBranchGroup(); for my $file ('dependency-data-common', "dependency-data-$branchGroup") { my $dependencyFile = $metadataModule->fullpath('source') . "/$file"; my $dependencies = pretend_open($dependencyFile) or die "Unable to open $dependencyFile: $!"; debug (" -- Reading dependencies from $dependencyFile"); $dependencyResolver->readDependencyData($dependencies); close $dependencies; } my @reorderedModules = $dependencyResolver->resolveDependencies(@modules); return @reorderedModules; }; if ($@) { warning (" r[b[*] Problems encountered trying to sort modules into correct order:"); warning (" r[b[*] $@"); warning (" r[b[*] Will attempt to continue."); } return @modules; } # Runs all update, build, install, etc. phases. Basically this *is* the # script. # The metadata module must already have performed its update by this point. sub runAllModulePhases { my $self = shift; my $ctx = $self->context(); my @modules = $self->modules(); if ($ctx->getOption('print-modules')) { for my $m (@modules) { say ((" " x ($m->getOption('#dependency-level', 'module') // 0)), "$m"); } return 0; # Abort execution early! } # Add to global module list now that we've filtered everything. $ctx->addModule($_) foreach @modules; my $runMode = $self->runMode(); if ($runMode eq 'query') { my $queryMode = $ctx->getOption('query', 'module'); # Default to ->getOption as query method. # $_[0] is short name for first param. my $query = sub { $_[0]->getOption($queryMode) }; $query = sub { $_[0]->fullpath('source') } if $queryMode eq 'source-dir'; $query = sub { $_[0]->fullpath('build') } if $queryMode eq 'build-dir'; $query = sub { $_[0]->installationPath() } if $queryMode eq 'install-dir'; $query = sub { $_[0]->fullProjectPath() } if $queryMode eq 'project-path'; $query = sub { ($_[0]->scm()->_determinePreferredCheckoutSource())[0] // '' } if $queryMode eq 'branch'; if (@modules == 1) { # No leading module name, just the value say $query->($modules[0]); } else { for my $m (@modules) { say "$m: ", $query->($m); } } return 0; } my $result; if ($runMode eq 'build') { # No packages to install, we're in build mode # What we're going to do is fork another child to perform the source # updates while we build. Setup for this first by initializing some # shared memory. my $ipc = 0; my $updateOptsSub = sub { my ($k, $v) = @_; $ctx->setPersistentOption($k, $v); }; if ($ctx->getOption('async')) { $ipc = ksb::IPC::Pipe->new(); $ipc->setPersistentOptionHandler($updateOptsSub); } if (!$ipc) { $ipc = ksb::IPC::Null->new(); $ipc->setPersistentOptionHandler($updateOptsSub); whisper ("Using no IPC mechanism\n"); note ("\n b[<<< Update Process >>>]\n"); $result = _handle_updates ($ipc, $ctx); note (" b[<<< Build Process >>>]\n"); $result = _handle_build ($ipc, $ctx) || $result; } else { $result = _handle_async_build ($ipc, $ctx); $ipc->outputPendingLoggedMessages() if debugging(); } } elsif ($runMode eq 'install') { $result = _handle_install ($ctx); } elsif ($runMode eq 'uninstall') { $result = _handle_uninstall ($ctx); } _cleanup_log_directory($ctx) if $ctx->getOption('purge-old-logs'); _output_failed_module_lists($ctx); # Record all failed modules. Unlike the 'resume-list' option this doesn't # include any successfully-built modules in between failures. my $failedModules = join(',', map { "$_" } $ctx->listFailedModules()); if ($failedModules) { # We don't clear the list of failed modules on success so that # someone can build one or two modules and still use # --rebuild-failures $ctx->setPersistentOption('global', 'last-failed-module-list', $failedModules); } # env driver is just the ~/.config/kde-env-*.sh, session driver is that + ~/.xsession if ($ctx->getOption('install-environment-driver') || $ctx->getOption('install-session-driver')) { _installCustomSessionDriver($ctx); } my $color = 'g[b['; $color = 'r[b[' if $result; info ("${color}", $result ? ":-(" : ":-)") unless pretending(); return $result; } # Method: finish # # Exits the script cleanly, including removing any lock files created. # # Parameters: # ctx - Required; BuildContext to use. # [exit] - Optional; if passed, is used as the exit code, otherwise 0 is used. sub finish { my $self = shift; my $ctx = $self->context(); my $exitcode = shift // 0; if (pretending() || $self->{_base_pid} != $$) { # Abort early if pretending or if we're not the same process # that was started by the user (e.g. async mode, forked pipe-opens exit $exitcode; } $ctx->closeLock(); $ctx->storePersistentOptions(); my $logdir = $ctx->getLogDir(); note ("Your logs are saved in y[$logdir]"); exit $exitcode; } ### Package-internal helper functions. # Reads a "line" from a file. This line is stripped of comments and extraneous # whitespace. Also, backslash-continued multiple lines are merged into a single # line. # # First parameter is the reference to the filehandle to read from. # Returns the text of the line. sub _readNextLogicalLine { my $fileReader = shift; while($_ = $fileReader->readLine()) { # Remove trailing newline chomp; # Replace \ followed by optional space at EOL and try again. if(s/\\\s*$//) { $_ .= $fileReader->readLine(); redo; } s/#.*$//; # Remove comments next if /^\s*$/; # Skip blank lines return $_; } return undef; } # Takes an input line, and extracts it into an option name, and simplified # value. The value has "false" converted to 0, white space simplified (like in # Qt), and tildes (~) in what appear to be path-like entries are converted to # the home directory path. # # First parameter is the build context (used for translating option values). # Second parameter is the line to split. # Return value is (option-name, option-value) sub _splitOptionAndValue { my $ctx = assert_isa(shift, 'ksb::BuildContext'); my $input = shift; my $optionRE = qr/\$\{([a-zA-Z0-9-]+)\}/; # The option is the first word, followed by the # flags on the rest of the line. The interpretation # of the flags is dependent on the option. my ($option, $value) = ($input =~ /^\s* # Find all spaces ([-\w]+) # First match, alphanumeric, -, and _ # (?: ) means non-capturing group, so (.*) is $value # So, skip spaces and pick up the rest of the line. (?:\s+(.*))?$/x); $value //= ''; # Simplify whitespace. $value =~ s/\s+$//; $value =~ s/^\s+//; $value =~ s/\s+/ /g; # Check for false keyword and convert it to Perl false. $value = 0 if lc($value) eq 'false'; # Replace reference to global option with their value. # The regex basically just matches ${option-name}. my ($sub_var_name) = ($value =~ $optionRE); while ($sub_var_name) { my $sub_var_value = $ctx->getOption($sub_var_name) || ''; if(!$ctx->hasOption($sub_var_name)) { warning (" *\n * WARNING: $sub_var_name is not set at line y[$.]\n *"); ## TODO: filename is missing } debug ("Substituting \${$sub_var_name} with $sub_var_value"); $value =~ s/\$\{$sub_var_name\}/$sub_var_value/g; # Replace other references as well. Keep this RE up to date with # the other one. ($sub_var_name) = ($value =~ $optionRE); } # Replace tildes with home directory. 1 while ($value =~ s"(^|:|=)~/"$1$ENV{'HOME'}/"); return ($option, $value); } # Ensures that the given ModuleSet has at least a valid repository and # use-modules setting based on the given BuildContext. sub _validateModuleSet { my ($ctx, $moduleSet) = @_; my $name = $moduleSet->name() || 'unnamed'; my $rcSources = _getModuleSources($moduleSet); # re-read option from module set since it may be pre-set my $selectedRepo = $moduleSet->getOption('repository'); if (!$selectedRepo) { error (<getOption('git-repository-base'); if ($selectedRepo ne KDE_PROJECT_ID && not exists $repoSet->{$selectedRepo}) { my $projectID = KDE_PROJECT_ID; my $moduleSetName = $moduleSet->name(); my $moduleSetId = $moduleSetName ? "module-set ($moduleSetName)" : "module-set"; error (<isa('ksb::BuildContext') ? 'global' : $module->isa('ksb::ModuleSet') ? 'module-set' : $module->isa('ksb::Module') ? 'module' : 'options'; # Just look for an end marker if terminator not provided. $endRE //= qr/^end[\w\s]*$/; _markModuleSource($module, $fileReader->currentFilename() . ":$."); # Read in each option while (($_ = _readNextLogicalLine($fileReader)) && ($_ !~ $endRE)) { my $current_file = $fileReader->currentFilename(); # Sanity check, make sure the section is correctly terminated if(/^(module\b|options\b)/) { error ("Invalid configuration file $current_file at line $.\nAdd an 'end $endWord' before " . "starting a new module.\n"); die make_exception('Config', "Invalid file $current_file"); } my ($option, $value) = _splitOptionAndValue($ctx, $_); eval { $module->setOption($option, $value); }; if (my $err = $@) { if (blessed($err) && $err->isa('ksb::BuildException::Config')) { my $msg = "$current_file:$.: " . $err->message(); my $explanation = $err->optionUsageExplanation(); $msg = $msg . "\n" . $explanation if $explanation; $err->setMessage($msg); } die; # re-throw } } return $module; } # Marks the given OptionsBase subclass (i.e. Module or ModuleSet) as being # read in from the given string (filename:line). An OptionsBase can be # tagged under multiple files. sub _markModuleSource { my ($optionsBase, $configSource) = @_; my $key = '#defined-at'; my $sourcesRef = $optionsBase->hasOption($key) ? $optionsBase->getOption($key) : []; push @$sourcesRef, $configSource; $optionsBase->setOption($key, $sourcesRef); } # Returns rcfile sources for given OptionsBase (comma-separated). sub _getModuleSources { my $optionsBase = shift; my $key = '#defined-at'; my $sourcesRef = $optionsBase->getOption($key) || []; return join(', ', @$sourcesRef); } # Reads in a "moduleset". # # First parameter is the build context. # Second parameter is the filehandle to the config file to read from. # Third parameter is the ksb::ModuleSet to use. # # Returns the ksb::ModuleSet passed in with read-in options set, which may need # to be further expanded (see ksb::ModuleSet::convertToModules). sub _parseModuleSetOptions { my ($ctx, $fileReader, $moduleSet) = @_; $moduleSet = _parseModuleOptions($ctx, $fileReader, $moduleSet, qr/^end\s+module(-?set)?$/); if ($moduleSet->getOption('repository') eq KDE_PROJECT_ID && !$moduleSet->isa('ksb::ModuleSet::KDEProjects')) { # Perl-specific note! re-blessing the module set into the right 'class' # You'd probably have to construct an entirely new object and copy the # members over in other languages. bless $moduleSet, 'ksb::ModuleSet::KDEProjects'; } return $moduleSet; } # Function: _readConfigurationOptions # # Reads in the settings from the configuration, passed in as an open # filehandle. # # Phase: # initialization - Do not call from this function. # # Parameters: # ctx - The to update based on the configuration read and # any pending command-line options (see cmdlineGlobalOptions). # # filehandle - The I/O object to read from. Must handle _eof_ and _readline_ # methods (e.g. subclass). # # deferredOptions - An out parameter: a hashref holding the options set by any # 'options' blocks read in by this function. Each key (identified by the name # of the 'options' block) will point to a hashref value holding the options to # apply. # # Returns: # @module - Heterogeneous list of and defined in the # configuration file. No module sets will have been expanded out (either # kde-projects or standard sets). # # Throws: # - Config exceptions. sub _readConfigurationOptions { my $ctx = assert_isa(shift, 'ksb::BuildContext'); my $fh = shift; my $deferredOptionsRef = shift; my @module_list; my $rcfile = $ctx->rcFile(); my ($option, %readModules); my $fileReader = ksb::RecursiveFH->new($rcfile); $fileReader->addFile($fh, $rcfile); # Read in global settings while ($_ = $fileReader->readLine()) { s/#.*$//; # Remove comments s/^\s*//; # Remove leading whitespace next if (/^\s*$/); # Skip blank lines # First command in .kdesrc-buildrc should be a global # options declaration, even if none are defined. if (not /^global\s*$/) { error ("Invalid configuration file: $rcfile."); error ("Expecting global settings section at b[r[line $.]!"); die make_exception('Config', 'Missing global section'); } # Now read in each global option. _parseModuleOptions($ctx, $fileReader, $ctx); last; } my $using_default = 1; my %seenModules; # NOTE! *not* module-sets, *just* modules. my %seenModuleSets; # and vice versa -- named sets only though! my %seenModuleSetItems; # To track option override modules. # Now read in module settings while ($_ = $fileReader->readLine()) { s/#.*$//; # Remove comments s/^\s*//; # Remove leading whitespace next if (/^\s*$/); # Skip blank lines # Get modulename (has dash, dots, slashes, or letters/numbers) my ($type, $modulename) = /^(options|module)\s+([-\/\.\w]+)\s*$/; my $newModule; # 'include' directives can change the current file, so check where we're at $rcfile = $fileReader->currentFilename(); # Module-set? if (not $modulename) { my $moduleSetRE = qr/^module-set\s*([-\/\.\w]+)?\s*$/; ($modulename) = m/$moduleSetRE/; # modulename may be blank -- use the regex directly to match if (not /$moduleSetRE/) { error ("Invalid configuration file $rcfile!"); error ("Expecting a start of module section at r[b[line $.]."); die make_exception('Config', 'Ungrouped/Unknown option'); } if ($modulename && exists $seenModuleSets{$modulename}) { error ("Duplicate module-set $modulename at $rcfile:$."); die make_exception('Config', "Duplicate module-set $modulename defined at $rcfile:$."); } if ($modulename && exists $seenModules{$modulename}) { error ("Name $modulename for module-set at $rcfile:$. is already in use on a module"); die make_exception('Config', "Can't re-use name $modulename for module-set defined at $rcfile:$."); } # A moduleset can give us more than one module to add. $newModule = _parseModuleSetOptions($ctx, $fileReader, ksb::ModuleSet->new($ctx, $modulename || "")); # Save 'use-modules' entries so we can see if later module decls # are overriding/overlaying their options. my @moduleSetItems = $newModule->moduleNamesToFind(); @seenModuleSetItems{@moduleSetItems} = ($newModule) x scalar @moduleSetItems; $seenModuleSets{$modulename} = $newModule if $modulename; } # Duplicate module entry? (Note, this must be checked before the check # below for 'options' sets) elsif (exists $seenModules{$modulename} && $type ne 'options') { error ("Duplicate module declaration b[r[$modulename] on line $. of $rcfile"); die make_exception('Config', "Duplicate module $modulename declared at $rcfile:$."); } # Module/module-set options overrides elsif ($type eq 'options') { my $options = _parseModuleOptions($ctx, $fileReader, ksb::OptionsBase->new()); $deferredOptionsRef->{$modulename} = $options->{options}; next; # Don't add to module list } # Must follow 'options' handling elsif (exists $seenModuleSets{$modulename}) { error ("Name $modulename for module at $rcfile:$. is already in use on a module-set"); die make_exception('Config', "Can't re-use name $modulename for module defined at $rcfile:$."); } else { $newModule = _parseModuleOptions($ctx, $fileReader, ksb::Module->new($ctx, $modulename)); $seenModules{$modulename} = $newModule; } push @module_list, $newModule; $using_default = 0; } while (my ($name, $moduleSet) = each %seenModuleSets) { _validateModuleSet($ctx, $moduleSet); } # If the user doesn't ask to build any modules, build a default set. # The good question is what exactly should be built, but oh well. if ($using_default) { warning (" b[y[*] There do not seem to be any modules to build in your configuration."); return (); } return @module_list; } # Exits out of kdesrc-build, executing the user's preferred shell instead. The # difference is that the environment variables should be as set in kdesrc-build # instead of as read from .bashrc and friends. # # You should pass in the options to run the program with as a list. # # Meant to implement the --run command line option. sub _executeCommandLineProgram { my ($program, @args) = @_; if (!$program) { error ("You need to specify a program with the --run option."); exit 1; # Can't use finish here. } if (($< != $>) && ($> == 0)) { error ("kdesrc-build will not run a program as root unless you really are root."); exit 1; } debug ("Executing b[r[$program] ", join(' ', @args)); exit 0 if pretending(); exec $program, @args or do { # If we get to here, that sucks, but don't continue. error ("Error executing $program: $!"); exit 1; }; } # Function: _split_url # # Subroutine to split a url into a protocol and host sub _split_url { my $url = shift; my ($proto, $host) = ($url =~ m|([^:]*)://([^/]*)/|); return ($proto, $host); } # Function: _check_for_ssh_agent # # Checks if we are supposed to use ssh agent by examining the environment, and # if so checks if ssh-agent has a list of identities. If it doesn't, we run # ssh-add (with no arguments) and inform the user. This can be controlled with # the disable-agent-check parameter. # # Parameters: # 1. Build context sub _check_for_ssh_agent { my $ctx = assert_isa(shift, 'ksb::BuildContext'); # Don't bother with all this if the user isn't even using SSH. return 1 if pretending(); my @svnServers = grep { $_->scmType() eq 'svn' } ($ctx->modulesInPhase('update')); my @gitServers = grep { $_->scmType() eq 'git' } ($ctx->modulesInPhase('update')); my @sshServers = grep { my ($proto, $host) = _split_url($_->getOption('svn-server')); # Check if ssh is explicitly used in the proto, or if the host is the # developer main svn. (defined $proto && $proto =~ /ssh/) || (defined $host && $host =~ /^svn\.kde\.org/); } @svnServers; push @sshServers, grep { # Check for git+ssh:// or git@git.kde.org:/path/etc. my $repo = $_->getOption('repository'); ($repo =~ /^git\+ssh:\/\//) || ($repo =~ /^[a-zA-Z0-9_.]+@.*:\//); } @gitServers; return 1 if (not @sshServers) or $ctx->getOption('disable-agent-check'); whisper ("\tChecking for SSH Agent") if (scalar @sshServers); # We're using ssh to download, see if ssh-agent is running. return 1 unless exists $ENV{'SSH_AGENT_PID'}; my $pid = $ENV{'SSH_AGENT_PID'}; # It's supposed to be running, let's see if there exists the program with # that pid (this check is linux-specific at the moment). if (-d "/proc" and not -e "/proc/$pid") { warning ("r[ *] SSH Agent is enabled, but y[doesn't seem to be running]."); warning ("Since SSH is used to download from Subversion you may want to see why"); warning ("SSH Agent is not working, or correct the environment variable settings."); return 0; } # The agent is running, but does it have any keys? We can't be more specific # with this check because we don't know what key is required. my $noKeys = 0; filter_program_output(sub { $noKeys ||= /no identities/ }, 'ssh-add', '-l'); if ($noKeys) { # Use print so user can't inadvertently keep us quiet about this. print ksb::Debug::colorize (<getOption('ssh-identity-file'); push (@commandLine, $identFile) if $identFile; my $result = system (@commandLine); if ($result) # Run this code for both death-by-signal and nonzero return { my $rcfile = $ctx->rcFile(); print "\nUnable to add SSH identity, aborting.\n"; print "If you don't want kdesrc-build to check in the future,\n"; print ksb::Debug::colorize ("Set the g[disable-agent-check] option to g[true] in your $rcfile.\n\n"); return 0; } } return 1; } # Function: _handle_updates # # Subroutine to update a list of modules. # # Parameters: # 1. IPC module to pass results to. # 2. Build Context, which will be used to determine the module update list. # # The ipc parameter contains an object that is responsible for communicating # the status of building the modules. This function must account for every # module in $ctx's update phase to the ipc object before returning. # # Returns 0 on success, non-zero on error. sub _handle_updates { my ($ipc, $ctx) = @_; my $kdesrc = $ctx->getSourceDir(); my @update_list = $ctx->modulesInPhase('update'); # No reason to print out the text if we're not doing anything. if (!@update_list) { $ipc->sendIPCMessage(ksb::IPC::ALL_UPDATING, "update-list-empty"); $ipc->sendIPCMessage(ksb::IPC::ALL_DONE, "update-list-empty"); return 0; } if (not _check_for_ssh_agent($ctx)) { $ipc->sendIPCMessage(ksb::IPC::ALL_FAILURE, "ssh-failure"); return 1; } if (not -e $kdesrc) { whisper ("KDE source download directory doesn't exist, creating.\n"); if (not super_mkdir ($kdesrc)) { error ("Unable to make directory r[$kdesrc]!"); $ipc->sendIPCMessage(ksb::IPC::ALL_FAILURE, "no-source-dir"); return 1; } } # Once at this point, any errors we get should be limited to a module, # which means we can tell the build thread to start. $ipc->sendIPCMessage(ksb::IPC::ALL_UPDATING, "starting-updates"); my $hadError = 0; foreach my $module (@update_list) { $ipc->setLoggedModule($module->name()); # Note that this must be in this order to avoid accidentally not # running ->update() from short-circuiting if an error is noted. $hadError = !$module->update($ipc, $ctx) || $hadError; } $ipc->sendIPCMessage(ksb::IPC::ALL_DONE, "had_errors: $hadError"); return $hadError; } # Builds the given module. # # Return value is the failure phase, or 0 on success. sub _buildSingleModule { my ($ipc, $ctx, $module, $startTimeRef) = @_; $ctx->resetEnvironment(); $module->setupEnvironment(); my $fail_count = $module->getPersistentOption('failure-count') // 0; my ($resultStatus, $message) = $ipc->waitForModule($module); $ipc->forgetModule($module); if ($resultStatus eq 'failed') { error ("\tUnable to update r[$module], build canceled."); $module->setPersistentOption('failure-count', ++$fail_count); return 'update'; } elsif ($resultStatus eq 'success') { note ("\tSource update complete for g[$module]: $message"); } # Skip actually building a module if the user has selected to skip # builds when the source code was not actually updated. But, don't skip # if we didn't successfully build last time. elsif ($resultStatus eq 'skipped' && !$module->getOption('build-when-unchanged') && $fail_count == 0) { note ("\tSkipping g[$module], its source code has not changed."); return 0; } elsif ($resultStatus eq 'skipped') { note ("\tNo changes to g[$module] source, proceeding to build."); } $$startTimeRef = time; $fail_count = $module->build() ? 0 : $fail_count + 1; $module->setPersistentOption('failure-count', $fail_count); return $fail_count > 0 ? 'build' : 0; } # Function: _handle_build # # Subroutine to handle the build process. # # Parameters: # 1. IPC object to receive results from. # 2. Build Context, which is used to determine list of modules to build. # # If the packages are not already checked-out and/or updated, this # subroutine WILL NOT do so for you. # # This subroutine assumes that the source directory has already been set up. # It will create the build directory if it doesn't already exist. # # If $builddir/$module/.refresh-me exists, the subroutine will # completely rebuild the module (as if --refresh-build were passed for that # module). # # Returns 0 for success, non-zero for failure. sub _handle_build { my ($ipc, $ctx) = @_; my @build_done; my @modules = $ctx->modulesInPhase('build'); my $result = 0; # No reason to print building messages if we're not building. return 0 if scalar @modules == 0; # Check for absolutely essential programs now. if (!_checkForEssentialBuildPrograms($ctx) && !exists $ENV{KDESRC_BUILD_IGNORE_MISSING_PROGRAMS}) { error (" r[b[*] Aborting now to save a lot of wasted time."); error (" y[b[*] export KDESRC_BUILD_IGNORE_MISSING_PROGRAMS=1 and re-run (perhaps with --no-src)"); error (" r[b[*] to continue anyways. If this check was in error please report a bug against"); error (" y[b[*] kdesrc-build at https://bugs.kde.org/"); return 1; } # IPC queue should have a message saying whether or not to bother with the # build. $ipc->waitForStreamStart(); $ctx->unsetPersistentOption('global', 'resume-list'); my $outfile = pretending() ? '/dev/null' : $ctx->getLogDir() . '/build-status'; open (STATUS_FILE, '>', $outfile) or do { error (<statusViewer(); my $i = 1; $statusViewer->numberModulesTotal(scalar @modules); while (my $module = shift @modules) { my $moduleName = $module->name(); my $moduleSet = $module->moduleSet()->name(); my $modOutput = $moduleName; if (debugging(ksb::Debug::WHISPER)) { $modOutput .= " (build system " . $module->buildSystemType() . ")" } $moduleSet = " from g[$moduleSet]" if $moduleSet; note ("Building g[$modOutput]$moduleSet ($i/$num_modules)"); my $start_time = time; my $failedPhase = _buildSingleModule($ipc, $ctx, $module, \$start_time); my $elapsed = prettify_seconds(time - $start_time); if ($failedPhase) { # FAILURE $ctx->markModulePhaseFailed($failedPhase, $module); print STATUS_FILE "$module: Failed on $failedPhase after $elapsed.\n"; if ($result == 0) { # No failures yet, mark this as resume point my $moduleList = join(', ', map { "$_" } ($module, @modules)); $ctx->setPersistentOption('global', 'resume-list', $moduleList); } $result = 1; if ($module->getOption('stop-on-failure')) { note ("\n$module didn't build, stopping here."); return 1; # Error } $statusViewer->numberModulesFailed(1 + $statusViewer->numberModulesFailed); } else { # Success print STATUS_FILE "$module: Succeeded after $elapsed.\n"; push @build_done, $moduleName; # Make it show up as a success $statusViewer->numberModulesSucceeded(1 + $statusViewer->numberModulesSucceeded); } $i++; } continue # Happens at the end of each loop and on next { print "\n"; # Space things out } if ($outfile) { close STATUS_FILE; # Update the symlink in latest to point to this file. my $logdir = $ctx->getSubdirPath('log-dir'); if (-l "$logdir/latest/build-status") { safe_unlink("$logdir/latest/build-status"); } symlink($outfile, "$logdir/latest/build-status"); } info ("<<< g[PACKAGES SUCCESSFULLY BUILT] >>>") if scalar @build_done > 0; my $successes = scalar @build_done; # TODO: l10n my $mods = $successes == 1 ? 'module' : 'modules'; if (not pretending()) { # Print out results, and output to a file my $kdesrc = $ctx->getSourceDir(); open BUILT_LIST, ">$kdesrc/successfully-built"; foreach my $module (@build_done) { info ("$module") if $successes <= 10; print BUILT_LIST "$module\n"; } close BUILT_LIST; info ("Built g[$successes] $mods") if $successes > 10; } else { # Just print out the results if ($successes <= 10) { info ('g[', join ("]\ng[", @build_done), ']'); } else { info ("Built g[$successes] $mods") if $successes > 10; } } info (' '); # Space out nicely return $result; } # Function: _handle_async_build # # This subroutine special-cases the handling of the update and build phases, by # performing them concurrently (where possible), using forked processes. # # Only one thread or process of execution will return from this procedure. Any # other processes will be forced to exit after running their assigned module # phase(s). # # We also redirect ksb::Debug output messages to be sent to a single process # for display on the terminal instead of allowing them all to interrupt each # other. # # Parameters: # 1. IPC Object to use for sending/receiving update/build status. It must be # an object type that supports IPC concurrency (e.g. IPC::Pipe). # 2. Build Context to use, from which the module lists will be determined. # # Returns 0 on success, non-zero on failure. sub _handle_async_build { # The exact method for async is that two children are forked. One child # is a source update process. The other child is a monitor process which will # hold status updates from the update process so that the updates may # happen without waiting for us to be ready to read. my ($ipc, $ctx) = @_; print "\n"; # Space out from metadata messages. my $result = 0; my $monitorPid = fork; if ($monitorPid == 0) { # child my $updaterToMonitorIPC = ksb::IPC::Pipe->new(); my $updaterPid = fork; $SIG{INT} = sub { POSIX::_exit(EINTR); }; if ($updaterPid) { $0 = 'kdesrc-build-updater'; $updaterToMonitorIPC->setSender(); ksb::Debug::setIPC($updaterToMonitorIPC); POSIX::_exit (_handle_updates ($updaterToMonitorIPC, $ctx)); } else { $0 = 'kdesrc-build-monitor'; $ipc->setSender(); $updaterToMonitorIPC->setReceiver(); $ipc->setLoggedModule('#monitor#'); # This /should/ never be used... ksb::Debug::setIPC($ipc); POSIX::_exit (_handle_monitoring ($ipc, $updaterToMonitorIPC)); } } else { # Still the parent, let's do the build. $ipc->setReceiver(); $result = _handle_build ($ipc, $ctx); } $ipc->waitForEnd(); $ipc->close(); # Display a message for updated modules not listed because they were not # built. my $unseenModulesRef = $ipc->unacknowledgedModules(); if (%$unseenModulesRef) { note ("The following modules were updated but not built:"); foreach my $modulename (keys %$unseenModulesRef) { note ("\t$modulename"); } } # It's possible if build fails on first module that git or svn is still # running. Make them stop too. if (waitpid ($monitorPid, WNOHANG) == 0) { kill 'INT', $monitorPid; # Exit code is in $?. waitpid ($monitorPid, 0); $result = 1 if $? != 0; } return $result; } # Function: _handle_install # # Handles the installation process. Simply calls 'make install' in the build # directory, though there is also provision for cleaning the build directory # afterwards, or stopping immediately if there is a build failure (normally # every built module is attempted to be installed). # # Parameters: # 1. Build Context, from which the install list is generated. # # Return value is a shell-style success code (0 == success) sub _handle_install { my $ctx = assert_isa(shift, 'ksb::BuildContext'); my @modules = $ctx->modulesInPhase('install'); @modules = grep { $_->buildSystem()->needsInstalled() } (@modules); my $result = 0; for my $module (@modules) { $ctx->resetEnvironment(); $result = $module->install() || $result; if ($result && $module->getOption('stop-on-failure')) { note ("y[Stopping here]."); return 1; # Error } } return $result; } # Function: _handle_uninstall # # Handles the uninstal process. Simply calls 'make uninstall' in the build # directory, while assuming that Qt or CMake actually handles it. # # The order of the modules is often significant, and it may work better to # uninstall modules in reverse order from how they were installed. However this # code does not automatically reverse the order; modules are uninstalled in the # order determined by the build context. # # This function obeys the 'stop-on-failure' option supported by _handle_install. # # Parameters: # 1. Build Context, from which the uninstall list is generated. # # Return value is a shell-style success code (0 == success) sub _handle_uninstall { my $ctx = assert_isa(shift, 'ksb::BuildContext'); my @modules = $ctx->modulesInPhase('uninstall'); @modules = grep { $_->buildSystem()->needsInstalled() } (@modules); my $result = 0; for my $module (@modules) { $ctx->resetEnvironment(); $result = $module->uninstall() || $result; if ($result && $module->getOption('stop-on-failure')) { note ("y[Stopping here]."); return 1; # Error } } return $result; } # Function: _handle_monitoring # # This is the main subroutine for the monitoring process when using IPC::Pipe. # It reads in all status reports from the source update process and then holds # on to them. When the build process is ready to read information we send what # we have. Otherwise we're waiting on the update process to send us something. # # This convoluted arrangement is required to allow the source update # process to go from start to finish without undue interruption on it waiting # to write out its status to the build process (which is usually busy). # # Parameters: # 1. the IPC object to use to send to build process. # 2. the IPC object to use to receive from update process. # # Returns 0 on success, non-zero on failure. sub _handle_monitoring { my ($ipcToBuild, $ipcFromUpdater) = @_; my @msgs; # Message queue. # We will write to the build process and read from the update process. my $sendFH = $ipcToBuild->{fh} || croak_runtime('??? missing pipe to build proc'); my $recvFH = $ipcFromUpdater->{fh} || croak_runtime('??? missing pipe from monitor'); my $readSelector = IO::Select->new($recvFH); my $writeSelector = IO::Select->new($sendFH); # Start the loop. We will be waiting on either read or write ends. # Whenever select() returns we must check both sets. while ( my ($readReadyRef, $writeReadyRef) = IO::Select->select($readSelector, $writeSelector, undef)) { if (!$readReadyRef && !$writeReadyRef) { # Some kind of error occurred. return 1; } # Check for source updates first. if (@{$readReadyRef}) { undef $@; my $msg = eval { $ipcFromUpdater->receiveMessage(); }; # undef msg indicates EOF, so check for exception obj specifically die $@ if $@; # undef can be returned on EOF as well as error. EOF means the # other side is presumably done. if (! defined $msg) { $readSelector->remove($recvFH); last; # Select no longer needed, just output to build. } else { push @msgs, $msg; # We may not have been waiting for write handle to be ready if # we were blocking on an update from updater thread. $writeSelector->add($sendFH) unless $writeSelector->exists($sendFH); } } # Now check for build updates. if (@{$writeReadyRef}) { # If we're here the update is still going. If we have no messages # to send wait for that first. if (not @msgs) { $writeSelector->remove($sendFH); } else { # Send the message (if we got one). if (!$ipcToBuild->sendMessage(shift @msgs)) { error ("r[mon]: Build process stopped too soon! r[$!]"); return 1; } } } } # Send all remaining messages. while (@msgs) { if (!$ipcToBuild->sendMessage(shift @msgs)) { error ("r[mon]: Build process stopped too soon! r[$!]"); return 1; } } $ipcToBuild->close(); return 0; } # Function: _applyModuleFilters # # Applies any module-specific filtering that is necessary after reading command # line and rc-file options. (This is as opposed to phase filters, which leave # each module as-is but change the phases they operate as part of, this # function could remove a module entirely from the build). # # Used for --resume-{from,after} and --stop-{before,after}, but more could be # added in theory. # This subroutine supports --{resume,stop}-* for both modules and module-sets. # # Parameters: # ctx - in use. # @modules - List of or to apply filters on. # # Returns: # list of or with any inclusion/exclusion filters # applied. Do not assume this list will be a strict subset of the input list, # however the order will not change amongst the input modules. sub _applyModuleFilters { my $ctx = assert_isa(shift, 'ksb::BuildContext'); my @moduleList = @_; if (!$ctx->getOption('resume-from') && !$ctx->getOption('resume-after') && !$ctx->getOption('stop-before') && !$ctx->getOption('stop-after')) { debug ("No command-line filter seems to be present."); return @moduleList; } if ($ctx->getOption('resume-from') && $ctx->getOption('resume-after')) { # This one's an error. error (<getOption('stop-before') && $ctx->getOption('stop-after')) { # This one's an error. error (<getOption('resume-from') || $ctx->getOption('resume-after'); my $startIndex = scalar @moduleList; if ($resumePoint) { debug ("Looking for $resumePoint for --resume-* option"); # || 0 is a hack to force Boolean context. my $filterInclusive = $ctx->getOption('resume-from') || 0; my $found = 0; for (my $i = 0; $i < scalar @moduleList; $i++) { my $module = $moduleList[$i]; $found = $module->name() eq $resumePoint; if ($found) { $startIndex = $filterInclusive ? $i : $i + 1; $startIndex = min($startIndex, scalar @moduleList - 1); last; } } } else { $startIndex = 0; } my $stopPoint = $ctx->getOption('stop-before') || $ctx->getOption('stop-after'); my $stopIndex = 0; if ($stopPoint) { debug ("Looking for $stopPoint for --stop-* option"); # || 0 is a hack to force Boolean context. my $filterInclusive = $ctx->getOption('stop-before') || 0; my $found = 0; for (my $i = $startIndex; $i < scalar @moduleList; $i++) { my $module = $moduleList[$i]; $found = $module->name() eq $stopPoint; if ($found) { $stopIndex = $i - ($filterInclusive ? 1 : 0); last; } } } else { $stopIndex = scalar @moduleList - 1; } if ($startIndex > $stopIndex || scalar @moduleList == 0) { # Lost all modules somehow. croak_runtime("Unknown resume -> stop point $resumePoint -> $stopPoint."); } return @moduleList[$startIndex .. $stopIndex]; } # This defines the factory function needed for lower-level code to properly be # able to create ksb::Module objects from just the module name, while still # having the options be properly set and having the module properly tied into a # context. sub _defineNewModuleFactory { my ($self, $resolver) = @_; my $ctx = $self->context(); $self->{module_factory} = sub { # We used to need a special module-set to ignore virtual deps (they # would throw errors if the name did not exist). But, the resolver # handles that fine as well. return $resolver->resolveModuleIfPresent(shift); }; } # This function converts any 'l10n' references on the command line to return a l10n # module with the proper build system, scm type, etc. # # The languages are selected using global/kde-languages (which should be used # exclusively from the configuration file). sub _expandl10nModules { my ($ctx, @modules) = @_; my $l10n = 'l10n-kde4'; assert_isa($ctx, 'ksb::BuildContext'); # Only filter if 'l10n' is actually present in list. my @matches = grep {$_->name() =~ /^(?:$l10n|l10n)$/} @modules; my @langs = split(' ', $ctx->getOption('kde-languages')); return @modules if (!@matches || !@langs); my $l10nModule; for my $match (@matches) { # Remove all instances of l10n. @modules = grep {$_->name() ne $match->name()} @modules; # Save l10n module if user had it in config. We only save the first # one encountered though. $l10nModule //= $match; } # No l10n module? Just create one. $l10nModule //= ksb::Module->new($ctx, $l10n); whisper ("\tAdding languages ", join(';', @langs), " to build."); $l10nModule->setScmType('l10n'); my $scm = $l10nModule->scm(); # Add all required directories to the l10n module. Its buildsystem should # know to skip scripts and templates. $scm->setLanguageDirs(qw/scripts templates/, @langs); $l10nModule->setBuildSystem($scm); push @modules, $l10nModule; return @modules; } # Updates the built-in phase list for all Modules passed into this function in # accordance with the options set by the user. sub _updateModulePhases { whisper ("Filtering out module phases."); for my $module (@_) { if ($module->getOption('manual-update') || $module->getOption('no-svn') || $module->getOption('no-src')) { $module->phases()->clear(); next; } if ($module->getOption('manual-build')) { $module->phases()->filterOutPhase('build'); $module->phases()->filterOutPhase('test'); $module->phases()->filterOutPhase('install'); } $module->phases()->filterOutPhase('install') unless $module->getOption('install-after-build'); $module->phases()->addPhase('test') if $module->getOption('run-tests'); } return @_; } # This subroutine extract the value from options of the form --option=value, # which can also be expressed as --option value. # # The first parameter is the option that the user passed to the cmd line (e.g. # --prefix=/opt/foo). # The second parameter is a reference to the list of command line options. # # The return value is the value of the option (the list of options might be # shorter by 1, copy it if you don't want it to change), or undef if no value # was provided. sub _extractOptionValue { my ($option, $options_ref) = @_; if ($option =~ /=/) { my @value = split(/=/, $option); shift @value; # We don't need the first one, that the --option part. return if (scalar @value == 0); # If we have more than one element left in @value it's because the # option itself has an = in it, make sure it goes back in the answer. return join('=', @value); } return if scalar @{$options_ref} == 0; return shift @{$options_ref}; } # Like _extractOptionValue, but throws an exception if the value is not # actually present, so you don't have to check for it yourself. If you do get a # return value, it will be defined to something. sub _extractOptionValueRequired { my ($option, $options_ref) = @_; my $returnValue = _extractOptionValue($option, $options_ref); if (not defined $returnValue) { croak_runtime("Option $option needs to be set to some value instead of left blank"); } return $returnValue; } # Function: _cleanup_log_directory # # This function removes log directories from old kdesrc-build runs. All log # directories not referenced by $log_dir/latest somehow are made to go away. # # Parameters: # 1. Build context. # # No return value. sub _cleanup_log_directory { my $ctx = assert_isa(shift, 'ksb::BuildContext'); my $logdir = $ctx->getSubdirPath('log-dir'); return 0 if ! -e "$logdir/latest"; # Could happen for error on first run... # This glob relies on the date being in the specific format YYYY-MM-DD-ID my @dirs = bsd_glob("$logdir/????-??-??-??/", GLOB_NOSORT); my @needed = _reachableModuleLogs("$logdir/latest"); # Convert a list to a hash lookup since Perl lacks a "list-has" my %needed_table; @needed_table{@needed} = (1) x @needed; my $length = scalar @dirs - scalar @needed; if ($length > 15) { # Arbitrary man is arbitrary note ("Removing y[b[$length] out of g[b[$#dirs] old log directories (this may take some time)..."); } elsif ($length > 0) { info ("Removing g[b[$length] out of g[b[$#dirs] old log directories..."); } for my $dir (@dirs) { my ($id) = ($dir =~ m/(\d\d\d\d-\d\d-\d\d-\d\d)/); safe_rmtree($dir) unless $needed_table{$id}; } } # Function: _output_failed_module_list # # Print out an error message, and a list of modules that match that error # message. It will also display the log file name if one can be determined. # The message will be displayed all in uppercase, with PACKAGES prepended, so # all you have to do is give a descriptive message of what this list of # packages failed at doing. # # No message is printed out if the list of failed modules is empty, so this # function can be called unconditionally. # # Parameters: # 1. Build Context # 2. Message to print (e.g. 'failed to foo') # 3. List of ksb::Modules that had failed to foo # # No return value. sub _output_failed_module_list { my ($ctx, $message, @fail_list) = @_; assert_isa($ctx, 'ksb::BuildContext'); $message = uc $message; # Be annoying if (@fail_list) { debug ("Message is $message"); debug ("\tfor ", join(', ', @fail_list)); } if (scalar @fail_list > 0) { my $homedir = $ENV{'HOME'}; my $logfile; warning ("\nr[b[<<< PACKAGES $message >>>]"); for my $module (@fail_list) { $logfile = $module->getOption('#error-log-file'); # async updates may cause us not to have a error log file stored. There's only # one place it should be though, take advantage of side-effect of log_command() # to find it. if (not $logfile) { my $logdir = $module->getLogDir() . "/error.log"; $logfile = $logdir if -e $logdir; } $logfile = "No log file" unless $logfile; $logfile =~ s|$homedir|~|; warning ("r[$module]") if pretending(); warning ("r[$module] - g[$logfile]") if not pretending(); } } } # Function: _output_failed_module_lists # # This subroutine reads the list of failed modules for each phase in the build # context and calls _output_failed_module_list for all the module failures. # # Parameters: # 1. Build context # # Return value: # None sub _output_failed_module_lists { my $ctx = assert_isa(shift, 'ksb::BuildContext'); # This list should correspond to the possible phase names (although # it doesn't yet since the old code didn't, TODO) for my $phase ($ctx->phases()->phases()) { my @failures = $ctx->failedModulesInPhase($phase); _output_failed_module_list($ctx, "failed to $phase", @failures); } # See if any modules fail continuously and warn specifically for them. my @super_fail = grep { ($_->getPersistentOption('failure-count') // 0) > 3 } (@{$ctx->moduleList()}); if (@super_fail) { warning ("\nThe following modules have failed to build 3 or more times in a row:"); warning ("\tr[b[$_]") foreach @super_fail; warning ("\nThere is probably a local error causing this kind of consistent failure, it"); warning ("is recommended to verify no issues on the system.\n"); } } # Function: _installTemplatedFile # # This function takes a given file and a build context, and installs it to a # given location while expanding out template entries within the source file. # # The template language is *extremely* simple: <% foo %> is replaced entirely # with the result of $ctx->getOption(foo, 'no-inherit'). If the result # evaluates false for any reason than an exception is thrown. No quoting of # any sort is used in the result, and there is no way to prevent expansion of # something that resembles the template format. # # Multiple template entries on a line will be replaced. # # The destination file will be created if it does not exist. If the file # already exists then an exception will be thrown. # # Error handling: Any errors will result in an exception being thrown. # # Parameters: # 1. Pathname to the source file (use absolute paths) # 2. Pathname to the destination file (use absolute paths) # 3. Build context to use for looking up template values # # Return value: There is no return value. sub _installTemplatedFile { my ($sourcePath, $destinationPath, $ctx) = @_; assert_isa($ctx, 'ksb::BuildContext'); open (my $input, '<', $sourcePath) or croak_runtime("Unable to open template source $sourcePath: $!"); open (my $output, '>', $destinationPath) or croak_runtime("Unable to open template output $destinationPath: $!"); while (!eof ($input)) { my $line = readline($input); if (!defined ($line)) { croak_runtime("Failed to read from $sourcePath at line $.: $!"); unlink($destinationPath); } # Some lines should only be present in the source as they aid with testing. next if $line =~ /kdesrc-build: filter/; $line =~ s { <% \s* # Template bracket and whitespace ([^\s%]+) # Capture variable name \s*%> # remaining whitespace and closing bracket } { $ctx->getOption($1, 'module') // croak_runtime("Invalid variable $1") }gxe; # Replace all matching expressions, use extended regexp w/ # comments, and replacement is Perl code to execute. (print $output $line) or croak_runtime("Unable to write line to $destinationPath at line $.: $!"); } } # Function: _installCustomFile # # This function installs a source file to a destination path, assuming the # source file is a "templated" source file (see also _installTemplatedFile), and # records a digest of the file actually installed. This function will overwrite # a destination if the destination is identical to the last-installed file. # # Error handling: Any errors will result in an exception being thrown. # # Parameters: # 1. Build context to use for looking up template values, # 2. The full path to the source file. # 3. The full path to the destination file (incl. name) # 4. The key name to use for searching/recording installed MD5 digest. # # Return value: There is no return value. sub _installCustomFile { use File::Copy qw(copy); my $ctx = assert_isa(shift, 'ksb::BuildContext'); my ($sourceFilePath, $destFilePath, $md5KeyName) = @_; my $baseName = basename($sourceFilePath); if (-e $destFilePath) { my $existingMD5 = $ctx->getPersistentOption('/digests', $md5KeyName) // ''; if (fileDigestMD5($destFilePath) ne $existingMD5) { if (!$ctx->getOption('#delete-my-settings')) { error ("\tr[*] Installing \"b[$baseName]\" would overwrite an existing file:"); error ("\tr[*] y[b[$destFilePath]"); error ("\tr[*] If this is acceptable, please delete the existing file and re-run,"); error ("\tr[*] or pass b[--delete-my-settings] and re-run."); return; } elsif (!pretending()) { copy ($destFilePath, "$destFilePath.kdesrc-build-backup"); } } } if (!pretending()) { _installTemplatedFile($sourceFilePath, $destFilePath, $ctx); $ctx->setPersistentOption('/digests', $md5KeyName, fileDigestMD5($destFilePath)); } } # Function: _installCustomSessionDriver # # This function installs the included sample .xsession and environment variable # setup files, and records the md5sum of the installed results. # # If a file already exists, then its md5sum is taken and if the same as what # was previously installed, is overwritten. If not the same, the original file # is left in place and the .xsession is instead installed to # .xsession-kdesrc-build # # Error handling: Any errors will result in an exception being thrown. # # Parameters: # 1. Build context to use for looking up template values, # # Return value: There is no return value. sub _installCustomSessionDriver { - use FindBin qw($RealBin); - use List::Util qw(first); use File::Copy qw(copy); my $ctx = assert_isa(shift, 'ksb::BuildContext'); - my @xdgDataDirs = split(':', $ENV{XDG_DATA_DIRS} || '/usr/local/share/:/usr/share/'); - my $xdgDataHome = $ENV{XDG_DATA_HOME} || "$ENV{HOME}/.local/share"; - # First we have to find the source - my @searchPaths = ($RealBin, map { "$_/apps/kdesrc-build" } ($xdgDataHome, @xdgDataDirs)); - - s{/+$}{} foreach @searchPaths; # Remove trailing slashes - s{//+}{/}g foreach @searchPaths; # Remove duplicate slashes - - my $envScript = first { -f $_ } ( - map { "$_/sample-kde-env-master.sh" } @searchPaths - ); - my $sessionScript = first { -f $_ } ( - map { "$_/sample-xsession.sh" } @searchPaths - ); + my $envScript = findDataResource('sample-kde-env-master.sh'); + my $sessionScript = findDataResource('sample-xsession.sh'); if (!$envScript || !$sessionScript) { warning ("b[*] Unable to find helper files to setup a login session."); warning ("b[*] You will have to setup login yourself, or install kdesrc-build properly."); return; } my $destDir = $ENV{XDG_CONFIG_HOME} || "$ENV{HOME}/.config"; super_mkdir($destDir) unless -d $destDir; _installCustomFile($ctx, $envScript, "$destDir/kde-env-master.sh", 'kde-env-master-digest'); _installCustomFile($ctx, $sessionScript, "$ENV{HOME}/.xsession", 'xsession-digest') if $ctx->getOption('install-session-driver'); if (!pretending()) { if ($ctx->getOption('install-session-driver') && !chmod (0744, "$ENV{HOME}/.xsession")) { error ("\tb[r[*] Error making b[~/.xsession] executable: $!"); error ("\tb[r[*] If this file is not executable you may not be able to login!"); }; } } # Function: _checkForEssentialBuildPrograms # # This subroutine checks for programs which are absolutely essential to the # *build* process and returns false if they are not all present. Right now this # just means qmake and cmake (although this depends on what modules are # actually present in the build context). # # Parameters: # 1. Build context # # Return value: # None sub _checkForEssentialBuildPrograms { my $ctx = assert_isa(shift, 'ksb::BuildContext'); return 1 if pretending(); my @buildModules = $ctx->modulesInPhase('build'); my %requiredPrograms; my %modulesRequiringProgram; foreach my $module ($ctx->modulesInPhase('build')) { my @progs = $module->buildSystem()->requiredPrograms(); # Deliberately used @, since requiredPrograms can return a list. @requiredPrograms{@progs} = 1; foreach my $prog (@progs) { $modulesRequiringProgram{$prog} //= { }; $modulesRequiringProgram{$prog}->{$module->name()} = 1; } } my $wasError = 0; for my $prog (keys %requiredPrograms) { my %requiredPackages = ( qmake => 'Qt', cmake => 'CMake', ); my $programPath = absPathToExecutable($prog); # qmake is not necessarily named 'qmake' if (!$programPath && $prog eq 'qmake') { $programPath = ksb::BuildSystem::QMake::absPathToQMake(); } if (!$programPath) { # Don't complain about Qt if we're building it... if ($prog eq 'qmake' && ( grep { $_->buildSystemType() eq 'Qt' } (@buildModules)) || pretending() ) { next; } $wasError = 1; my $reqPackage = $requiredPackages{$prog} || $prog; my @modulesNeeding = keys %{$modulesRequiringProgram{$prog}}; local $, = ', '; # List separator in output error (<<"EOF"); Unable to find r[b[$prog]. This program is absolutely essential for building the modules: y[@modulesNeeding]. Please ensure the development packages for $reqPackage are installed by using your distribution's package manager. You can also see the https://techbase.kde.org/Getting_Started/Build/Distributions page for information specific to your distribution (although watch for outdated information :( ). EOF } } return !$wasError; } # Function: _reachableModuleLogs # # Returns a list of module directory IDs that must be kept due to being # referenced from the "latest" symlink. # # This function may call itself recursively if needed. # # Parameters: # 1. The log directory under which to search for symlinks, including the "/latest" # part of the path. sub _reachableModuleLogs { my $logdir = shift; my @dirs; # A lexicalized var (my $foo) is required in face of recursiveness. opendir(my $fh, $logdir) or croak_runtime("Can't opendir $logdir: $!"); my $dir = readdir($fh); while(defined $dir) { if (-l "$logdir/$dir") { my $link = readlink("$logdir/$dir"); push @dirs, $link; } elsif ($dir !~ /^\.{1,2}$/) { # Skip . and .. directories (this is a great idea, trust me) push @dirs, _reachableModuleLogs("$logdir/$dir"); } $dir = readdir $fh; } closedir $fh; # Extract numeric IDs from directory names. @dirs = map { m/(\d{4}-\d\d-\d\d-\d\d)/ } (@dirs); # Convert to unique list by abusing hash keys. my %tempHash; @tempHash{@dirs} = (); return keys %tempHash; } # Installs the given subroutine as a signal handler for a set of signals which # could kill the program. # # First parameter is a reference to the sub to act as the handler. sub _installSignalHandlers { my $handlerRef = shift; my @signals = qw/HUP INT QUIT ABRT TERM PIPE/; @SIG{@signals} = ($handlerRef) x scalar @signals; } # Ensures that basic one-time setup to actually *use* installed software is # performed, including .kdesrc-buildrc setup if necessary. # # Returns the appropriate exitcode to pass to the exit function sub performInitialUserSetup { my $self = shift; return ksb::FirstRun::setupUserSystem(); } # Shows a help message and version. Does not exit. sub _showHelpMessage { my $scriptVersion = scriptVersion(); say < and others, and is distributed under the terms of the GNU GPL v2. This script automates the download, build, and install process for KDE software using the latest available source code. Configuration is controlled from "\$PWD/kdesrc-buildrc" or "~/.kdesrc-buildrc". See kdesrc-buildrc-sample for an example. Usage: \$ $0 [--options] [module names] All configured modules are built if none are listed. Important Options: --pretend Don't actually take major actions, instead describe what would be done. --no-src Don't update source code, just build/install. --src-only Only update the source code --refresh-build Start the build from scratch. --rc-file= Read configuration from filename instead of default. --initial-setup Installs Plasma env vars (~/.bashrc), required system pkgs, and a base kdesrc-buildrc. --resume-from= Skips modules until just before or after the given --resume-after= package, then operates as normal. --stop-before= Stops just before or after the given package is --stop-after= reached. --include-dependencies Also builds KDE-based dependencies of given modules. --stop-on-failure Stops the build as soon as a package fails to build. More docs at https://docs.kde.org/trunk5/en/extragear-utils/kdesrc-build/ Supported configuration options: https://go.kde.org/u/ksboptions Supported cmdline options: https://go.kde.org/u/ksbcmdline DONE # Look for indications this is the first run. if (! -e "./kdesrc-buildrc" && ! -e "$ENV{HOME}/.kdesrc-buildrc") { say <{context}; } sub metadataModule { my $self = shift; return $self->{metadata_module}; } sub runMode { my $self = shift; return $self->{run_mode}; } sub modules { my $self = shift; return @{$self->{modules}}; } 1; diff --git a/modules/ksb/FirstRun.pm b/modules/ksb/FirstRun.pm index 7cc52d5..b355fab 100644 --- a/modules/ksb/FirstRun.pm +++ b/modules/ksb/FirstRun.pm @@ -1,321 +1,322 @@ package ksb::FirstRun 0.10; use 5.014; use strict; use warnings; use File::Spec qw(splitpath); use ksb::BuildException; use ksb::Debug qw(colorize); use ksb::OSSupport; =head1 NAME ksb::FirstRun =head1 DESCRIPTION Performs initial-install setup, implementing the C<--initial-setup> option. B This module is supposed to be loadable even under minimal Perl environments as fielded in "minimal Docker container" forms of popular distros. =head1 SYNOPSIS - my $exitcode = ksb::FirstRun::setupUserSystem(); + # $baseDir is the data directory (either from source dir or as installed) + my $exitcode = ksb::FirstRun::setupUserSystem($baseDir); exit $exitcode; =cut sub setupUserSystem { my $baseDir = shift; my $os = ksb::OSSupport->new; eval { _installSystemPackages($os); _setupBaseConfiguration($baseDir); _setupBashrcFile(); }; if (had_an_exception($@)) { my $msg = $@->{message}; say colorize (" b[r[*] r[$msg]"); return 1; } return 0; } # Internal functions # Reads from the __DATA__ section below and dumps the contents in a hash keyed # by filename (the @@ part between each resource). my %packages; sub _readPackages { return \%packages if %packages; my $cur_file; my $cur_value; my $commit = sub { return unless $cur_file; $packages{$cur_file} = ($cur_value =~ s/ *$//r); $cur_value = ''; }; while(my $line = ) { next if $line =~ /^\s*#/ and $cur_file !~ /sample-rc/; chomp $line; my ($fname) = ($line =~ /^@@ *([^ ]+)$/); if ($fname) { $commit->(); $cur_file = $fname; } else { $cur_value .= "$line\n"; } } $commit->(); return \%packages; } sub _throw { my $msg = shift; die (make_exception('Setup', $msg)); } sub _installSystemPackages { my $os = shift; my $vendor = $os->vendorID; my $osVersion = $os->vendorVersion; print colorize(<> 8 == 0) { say colorize (" b[*] b[g[Looks like things went OK!]"); } else { say colorize (" r[b[*] Ran into an error with the installer!"); } } else { say colorize (" r[b[*] Whoa, I'm not familiar with your distribution, skipping"); } } sub _setupBaseConfiguration { my $baseDir = shift; if (-e "kdesrc-buildrc" || -e "$ENV{HOME}/.kdesrc-buildrc") { print colorize(</dev/null` || 4; $sampleRc =~ s/%\{num_cpus}/$numCpus/; $sampleRc =~ s/%\{base_dir}/$baseDir/; open my $sampleFh, '>', "$ENV{HOME}/.kdesrc-buildrc" or _throw("Couldn't open new ~/.kdesrc-buildrc: $!"); print $sampleFh $sampleRc or _throw("Couldn't write to ~/.kdesrc-buildrc: $!"); close $sampleFh or _throw("Error closing ~/.kdesrc-buildrc: $!"); } } sub _bashrcIsSetup { return 1; } sub _setupBashrcFile { if (_bashrcIsSetup()) { say colorize(<bestDistroMatch(@supportedDistros); say colorize (" Using installer for b[$bestVendor]"); my $version = $os->vendorVersion(); my @cmd; for my $opt ("$bestVendor/$version", "$bestVendor/unknown") { my $key = "cmd/install/$opt"; next unless exists $pkgsRef->{$key}; @cmd = split(' ', $pkgsRef->{$key}); last; } _throw("No installer for $bestVendor!") unless @cmd; # If not running as root already, add sudo unshift @cmd, 'sudo' if $> != 0; return @cmd; } sub _findBestVendorPackageList { my $os = shift; # Debian handles Ubuntu also my @supportedDistros = map { s{^pkg/([^/]+)/.*$}{$1}; $_ } grep { /^pkg\// } keys %{_readPackages()}; my $bestVendor = $os->bestDistroMatch(@supportedDistros); my $version = $os->vendorVersion(); say colorize (" Installing packages for b[$bestVendor]/b[$version]"); return _packagesForVendor($bestVendor, $version); } sub _packagesForVendor { my ($vendor, $version) = @_; my $packagesRef = _readPackages(); foreach my $opt ("pkg/$vendor/$version", "pkg/$vendor/unknown") { next unless exists $packagesRef->{$opt}; my @packages = split(' ', $packagesRef->{$opt}); return @packages; } return; } 1; __DATA__ @@ pkg/debian/unknown libyaml-libyaml-perl libio-socket-ssl-perl libjson-xs-perl git shared-mime-info cmake build-essential flex bison gperf libssl-dev intltool liburi-perl gettext @@ pkg/opensuse/unknown perl perl-IO-Socket-SSL perl-JSON perl-YAML-LibYAML git shared-mime-info make cmake libqt5-qtbase-common-devel libopenssl-devel intltool polkit-devel libqt5-qtbase-devel libqt5-qtimageformats-devel libqt5-qtmultimedia-devel libqt5-qtdeclarative-devel libqt5-qtx11extras-devel libqt5-qtxmlpatterns-devel libqt5-qtsvg-devel gperf gettext-runtime gettext-tools libxml2-devel libxml2-tools libxslt-devel docbook-xsl-stylesheets docbook_4 perl-URI libXrender-devel xcb-util-keysyms-devel flex bison libQt5Core-private-headers-devel libudev-devel libQt5WebKit5-devel libQt5WebKitWidgets-devel libQt5DesignerComponents5 libqt5-qttools-devel libSM-devel libattr-devel libboost_headers1_66_0-devel libQt5QuickControls2-devel libqt5-qtscript-devel wayland-devel libqt5-qtbase-private-headers-devel lmdb-devel libpng16-compat-devel giflib-devel ModemManager-devel # This pulls in so many other packages! :( NetworkManager-devel qrencode-devel @@ pkg/fedora/unknown git @@ pkg/gentoo/unknown dev-util/cmake dev-lang/perl @@ pkg/arch/unknown perl-json perl-yaml-libyaml perl-io-socket-ssl cmake gcc make qt5-base @@ cmd/install/debian/unknown apt-get -q -y --no-install-recommends install @@ cmd/install/opensuse/unknown zypper install -y --no-recommends @@ cmd/install/arch/unknown pacman -Sy --noconfirm @@ sample-rc # This file controls options to apply when configuring/building modules, and # controls which modules are built in the first place. # List of all options: https://go.kde.org/u/ksboptions global branch-group kf5-qt5 kdedir ~/kde-5 # Where to install KF5-based software # Uncomment this and edit value to choose a different Qt5 # qtdir /usr # Where to find Qt5 # Will pull in KDE-based dependencies only, to save you the trouble of # listing them all below include-dependencies true source-dir ~/kde/src build-dir ~/kde/build cmake-options -DCMAKE_BUILD_TYPE=RelWithDebInfo make-options -j%{num_cpus} end global # With base options set, the remainder of the file is used to define modules to build, in the # desired order, and set any module-specific options. # # Modules may be grouped into sets, and this is the normal practice. # # You can include other files inline using the "include" command. We do this here # to include files which are updated with kdesrc-build. include %{base_dir}/kf5-qt5-build-include # To change options for modules that have already been defined, use an # 'options' block options kcoreaddons make-options -j4 end options diff --git a/modules/ksb/Util.pm b/modules/ksb/Util.pm index 5a77947..ccbf684 100644 --- a/modules/ksb/Util.pm +++ b/modules/ksb/Util.pm @@ -1,670 +1,685 @@ -package ksb::Util 0.30; +package ksb::Util 0.35; # Useful utilities, which are exported into the calling module's namespace by default. use 5.014; # Needed for state keyword use strict; use warnings; use Scalar::Util qw(blessed); use File::Path qw(make_path); use File::Find; use Cwd qw(getcwd); use Errno qw(:POSIX); use Digest::MD5; use ksb::Debug; use ksb::Version qw(scriptVersion); use ksb::BuildException; use Exporter qw(import); # Use Exporter's import method our @EXPORT = qw(list_has assert_isa assert_in any unique_items absPathToExecutable fileDigestMD5 log_command disable_locale_message_translation split_quoted_on_whitespace safe_unlink safe_system p_chdir pretend_open safe_rmtree get_list_digest is_dir_empty - super_mkdir filter_program_output prettify_seconds); + super_mkdir filter_program_output prettify_seconds + findDataResource + ); # Function to work around a Perl language limitation. # First parameter is a reference to the list to search. ALWAYS. # Second parameter is the value to search for. # Returns true if the value is in the list sub list_has { my ($listRef, $value) = @_; my @list = @{$listRef}; return scalar grep { "$_" eq "$value" } (@list); } # Subroutine to return the path to the given executable based on the # current PATH. e.g. if you pass make you could get '/usr/bin/make'. If # the executable is not found undef is returned. # # This assumes that the module environment has already been updated since # binpath doesn't exactly correspond to $ENV{'PATH'}. sub absPathToExecutable { my $prog = shift; my @paths = split(/:/, $ENV{'PATH'}); # If it starts with a / the path is already absolute. return $prog if $prog =~ /^\//; for my $path (@paths) { return "$path/$prog" if (-x "$path/$prog"); } return undef; } # Throws an exception if the first parameter is not an object at all, or if # it is not an object of the type given by the second parameter (which # should be a string of the class name. There is no return value; sub assert_isa { my ($obj, $class) = @_; if (!blessed($obj) || !$obj->isa($class)) { croak_internal("$obj is not of type $class, but of type " . ref($obj)); } return $obj; } # Throws an exception if the first parameter is not included in the # provided list of possible alternatives. The list of alternatives must # be passed as a reference, as the second parameter. sub assert_in { my ($val, $listRef) = @_; if (!list_has($listRef, $val)) { croak_runtime("$val is not a permissible value for its argument"); } return $val; } # Subroutine to unlink the given symlink if global-pretend isn't set. sub safe_unlink { if (pretending()) { pretend ("\tWould have unlinked ", shift, "."); return 1; # Return true } return unlink (shift); } # Subroutine to execute the system call on the given list if the pretend # global option is not set. # # Returns the shell error code, so 0 means success, non-zero means failure. sub safe_system(@) { if (!pretending()) { whisper ("\tExecuting g['", join("' '", @_), "'"); return system (@_) >> 8; } pretend ("\tWould have run g['" . join("' '", @_) . "'"); return 0; # Return true } # Is exactly like "chdir", but it will also print out a message saying that # we're switching to the directory when debugging. sub p_chdir($) { my $dir = shift; debug ("\tcd g[$dir]\n"); chdir ($dir) or do { return 1 if pretending(); croak_runtime("Could not change to directory $dir: $!"); }; } # Helper subroutine to create a directory, including any parent # directories that may also need created. # Throws an exception on failure. See File::Path. sub super_mkdir { my $pathname = shift; state %createdPaths; if (pretending()) { if (!exists $createdPaths{$pathname} && ! -e $pathname) { pretend ("\tWould have created g[$pathname]"); } $createdPaths{$pathname} = 1; return 1; } else { make_path($pathname); return (-e $pathname) ? 1 : 0; } } # Calculates the MD5 digest of a file already on-disk. The digest is # returned as a hex string digest as from Digest::MD5::md5_hex # # First parameter: File name to read # Return value: hex string MD5 digest of file. # An exception is thrown if an error occurs reading the file. sub fileDigestMD5 { my $fileName = shift; my $md5 = Digest::MD5->new; open my $file, '<', $fileName or croak_runtime( "Unable to open $fileName: $!"); binmode($file); $md5->addfile($file); return $md5->hexdigest(); } # This function is intended to disable the message translation catalog # settings in the program environment, so that any child processes executed # will have their output untranslated (and therefore scrapeable). # # As such this should only be called for a forked child about to exec as # there is no easy way to undo this within the process. sub disable_locale_message_translation { # Ensure that program output is untranslated by setting 'C' locale. # We're really trying to affect the LC_MESSAGES locale category, but # LC_ALL is a catch-all for that (so needs to be unset if set). # # Note that the ONLY SUPPORTED way to pass file names, command-line # args, etc. to commands is under the UTF-8 encoding at this point, as # that is the only sane way for this en_US-based developer to handle # the task. Patches (likely using Encode::Locale) are accepted. :P $ENV{'LC_MESSAGES'} = 'C'; if ($ENV{'LC_ALL'}) { $ENV{'LANG'} = $ENV{'LC_ALL'}; # This is lower-priority "catch all" delete $ENV{'LC_ALL'}; } } # Returns an array of lines output from a program. Use this only if you # expect that the output will be short. # # Since there is no way to disambiguate no output from an error, this # function will call die on error, wrap in eval if this bugs you. # # First parameter is subroutine reference to use as a filter (this sub will # be passed a line at a time and should return true if the line should be # returned). If no filtering is desired pass 'undef'. # # Second parameter is the program to run (either full path or something # accessible in $PATH). # # All remaining arguments are passed to the program. # # Return value is an array of lines that were accepted by the filter. sub filter_program_output { my ($filterRef, $program, @args) = @_; $filterRef //= sub { return 1 }; # Default to all lines debug ("Slurping '$program' '", join("' '", @args), "'"); # Check early for whether an executable exists since otherwise # it is possible for our fork-open below to "succeed" (i.e. fork() # happens OK) and then fail when it gets to the exec(2) syscall. if (!absPathToExecutable($program)) { croak_runtime("Can't find $program in PATH!"); } my $execFailedError = "\t - kdesrc-build - exec failed!\n"; my $pid = open(my $childOutput, '-|'); croak_internal("Can't fork: $!") if ! defined($pid); if ($pid) { # parent my @lines = grep { &$filterRef; } (<$childOutput>); close $childOutput or do { # $! indicates a rather grievous error croak_internal("Unable to open pipe to read $program output: $!") if $!; # we can pass serious errors back to ourselves too. my $exitCode = $? >> 8; if ($exitCode == 99 && @lines >= 1 && $lines[0] eq $execFailedError) { croak_runtime("Failed to exec $program, is it installed?"); } # other errors might still be serious but don't need a backtrace if (pretending()) { whisper ("$program gave error exit code $exitCode"); } else { warning ("$program gave error exit code $exitCode"); } }; return @lines; } else { disable_locale_message_translation(); # We don't want stderr output on tty. open (STDERR, '>', '/dev/null') or close (STDERR); exec { $program } ($program, @args) or do { # Send a message back to parent print $execFailedError; exit 99; # Helper proc, so don't use finish(), just die }; } } # Subroutine to return a string suitable for displaying an elapsed time, # (like a stopwatch) would. The first parameter is the number of seconds # elapsed. sub prettify_seconds { my $elapsed = $_[0]; my $str = ""; my ($days,$hours,$minutes,$seconds,$fraction); $fraction = int (100 * ($elapsed - int $elapsed)); $elapsed = int $elapsed; $seconds = $elapsed % 60; $elapsed = int $elapsed / 60; $minutes = $elapsed % 60; $elapsed = int $elapsed / 60; $hours = $elapsed % 24; $elapsed = int $elapsed / 24; $days = $elapsed; $seconds = "$seconds.$fraction" if $fraction; my @str_list; for (qw(days hours minutes seconds)) { # Use a symbolic reference without needing to disable strict refs. # I couldn't disable it even if I wanted to because these variables # aren't global or localized global variables. my $value = eval "return \$$_;"; my $text = $_; $text =~ s/s$// if $value == 1; # Make singular push @str_list, "$value $text" if $value or $_ eq 'seconds'; } # Add 'and ' in front of last element if there was more than one. push @str_list, ("and " . pop @str_list) if (scalar @str_list > 1); $str = join (", ", @str_list); return $str; } # Subroutine to mark a file as being the error log for a module. This also # creates a symlink in the module log directory for easy viewing. # First parameter is the module in question. # Second parameter is the filename in the log directory of the error log. sub _setErrorLogfile { my $module = assert_isa(shift, 'ksb::Module'); my $logfile = shift; return unless $logfile; my $logdir = $module->getLogDir(); $module->setOption('#error-log-file', "$logdir/$logfile"); debug ("Logfile for $module is $logfile"); # Setup symlink in the module log directory pointing to the appropriate # file. Make sure to remove it first if it already exists. unlink("$logdir/error.log") if -l "$logdir/error.log"; if(-e "$logdir/error.log") { # Maybe it was a regular file? error ("r[b[ * Unable to create symlink to error log file]"); return; } symlink "$logfile", "$logdir/error.log"; } # Subroutine to run a command, optionally filtering on the output of the child # command. # # First parameter is the module object being built (for logging purposes # and such). # Second parameter is the name of the log file to use (relative to the log # directory). # Third parameter is a reference to an array with the command and its # arguments. i.e. ['command', 'arg1', 'arg2'] # # After the required three parameters you can pass a hash reference of # optional features: # 'callback' => a reference to a subroutine to have each line # of child output passed to. This output is not supposed to be printed # to the screen by the subroutine, normally the output is only logged. # However this is useful for e.g. munging out the progress of the build. # USEFUL: When there is no more output from the child, the callback will be # called with an undef string. (Not just empty, it is also undefined). # # 'no_translate' => any true value will cause a flag to be set to request # the executed child process to not translate (for locale purposes) its # output, so that it can be screen-scraped. # # The return value is the shell return code, so 0 is success, and non-zero is # failure. # # NOTE: This function has a special feature. If the command passed into the # argument reference is 'kdesrc-build', then log_command will, when it # forks, execute the subroutine named by the second parameter rather than # executing a child process. The subroutine should include the full package # name as well (otherwise the package containing log_command's implementation # is used). The remaining arguments in the list are passed to the # subroutine that is called. sub log_command { my ($module, $filename, $argRef, $optionsRef) = @_; assert_isa($module, 'ksb::Module'); my @command = @{$argRef}; $optionsRef //= { }; my $callbackRef = $optionsRef->{'callback'}; debug ("log_command(): Module $module, Command: ", join(' ', @command)); if (pretending()) { pretend ("\tWould have run g['" . join ("' '", @command) . "'"); return 0; } # Do this before we fork so we can see errors my $logpath = $module->getLogPath("$filename.log"); # Fork a child, with its stdout connected to CHILD. my $pid = open(CHILD, '-|'); if ($pid) { # Parent if (!$callbackRef && debugging()) { # If no other callback given, pass to debug() if debug-mode is on. while () { print ($_) if $_; } } if ($callbackRef) { &{$callbackRef}($_) while (); # Let callback know there is no more output. &{$callbackRef}(undef); } # This implicitly does a waitpid() as well close CHILD or do { if ($! == 0) { _setErrorLogfile($module, "$filename.log"); return $?; } return 1; }; return 0; } else { # Child. Note here that we need to avoid running our exit cleanup # handlers in here. For that we need POSIX::_exit. # Apply altered environment variables. $module->buildContext()->commitEnvironmentChanges(); $SIG{PIPE} = "IGNORE"; $SIG{INT} = sub { close (STDOUT); # This should be a pipe close (STDERR); POSIX::_exit(EINTR); }; # Redirect STDIN to /dev/null so that the handle is open but fails when # being read from (to avoid waiting forever for e.g. a password prompt # that the user can't see. open (STDIN, '<', "/dev/null") unless exists $ENV{'KDESRC_BUILD_USE_TTY'}; if ($callbackRef || debugging()) { open (STDOUT, "|tee $logpath") or do { error ("Error opening pipe to tee command."); # Don't abort, hopefully STDOUT still works. }; } else { open (STDOUT, '>', $logpath) or do { error ("Error $! opening log to $logpath!"); }; } # Make sure we log everything. open (STDERR, ">&STDOUT"); # Call internal function, name given by $command[1] if ($command[0] eq 'kdesrc-build') { # No colors! ksb::Debug::setColorfulOutput(0); debug ("Calling $command[1]"); my $cmd = $command[1]; splice (@command, 0, 2); # Remove first two elements. no strict 'refs'; # Disable restriction on symbolic subroutines. if (! &{$cmd}(@command)) # Call sub { POSIX::_exit (EINVAL); } POSIX::_exit (0); # Exit child process successfully. } # Don't leave empty output files, give an indication of the particular # command run. Use print to go to stdout. say "# kdesrc-build running: '", join("' '", @command), "'"; say "# from directory: ", getcwd(); # If a callback is set assume no translation can be permitted. disable_locale_message_translation() if $optionsRef->{'no_translate'}; # External command. exec (@command) or do { my $cmd_string = join(' ', @command); error (<($_) && return 1) foreach @{$listRef}; return 0; } # Returns unique items of the list. Order not guaranteed. sub unique_items { # See perlfaq4 my %seen; my @results = grep { ! $seen{$_}++; } @_; return @results; } # Subroutine to delete a directory and all files and subdirectories within. # Does nothing in pretend mode. An analog to "rm -rf" from Linux. # Requires File::Find module. # # First parameter: Path to delete # Returns boolean true on success, boolean false for failure. sub safe_rmtree { my $path = shift; # Pretty user-visible path my $user_path = $path; $user_path =~ s/^$ENV{HOME}/~/; my $delete_file_or_dir = sub { # $_ is the filename/dirname. return if $_ eq '.' or $_ eq '..'; if (-f $_ || -l $_) { unlink ($_) or croak_runtime("Unable to delete $File::Find::name: $!"); } elsif (-d $_) { rmdir ($File::Find::name) or croak_runtime("Unable to remove directory $File::Find::name: $!"); } }; if (pretending()) { pretend ("Would have removed all files/folders in $user_path"); return 1; } # Error out because we probably have a logic error even though it would # delete just fine. if (not -d $path) { error ("Cannot recursively remove $user_path, as it is not a directory."); return 0; } eval { $@ = ''; finddepth( # finddepth does a postorder traversal. { wanted => $delete_file_or_dir, no_chdir => 1, # We'll end up deleting directories, so prevent this. }, $path); }; if ($@) { error ("Unable to remove directory $user_path: $@"); return 0; } return 1; } # Returns a hash digest of the given options in the list. The return value is # base64-encoded at this time. # # Note: Don't be dumb and pass data that depends on execution state as the # returned hash is almost certainly not useful for whatever you're doing with # it. (i.e. passing a reference to a list is not helpful, pass the list itself) # # Parameters: List of scalar values to hash. # Return value: base64-encoded hash value. sub get_list_digest { use Digest::MD5 "md5_base64"; # Included standard with Perl 5.8 return md5_base64(@_); } # Utility function to see if a directory path is empty or not sub is_dir_empty { my $dir = shift; opendir my $dirh, $dir or return; # while-readdir needs Perl 5.12 while (readdir $dirh) { next if ($_ eq '.' || $_ eq '..'); closedir ($dirh); return; # not empty } closedir ($dirh); return 1; } -1; +# Returns path to a named data resource or throws an exception if it can't be +# found +sub findDataResource +{ + my $name = shift; + my $baseDir = $main::baseDataDir; + my $resPath = "$baseDir/$name"; + + croak_internal("Can't find resource $name") + unless -e $resPath; + return $resPath; +} + +1;