diff --git a/modules/ksb/Application.pm b/modules/ksb/Application.pm index cade22b..00a67d6 100644 --- a/modules/ksb/Application.pm +++ b/modules/ksb/Application.pm @@ -1,2705 +1,2691 @@ package ksb::Application 0.20; # Class: Application # # Contains the application-layer logic (i.e. creating a build context, reading # options, parsing command-line, etc.) use strict; use warnings; use 5.014; no if $] >= 5.018, 'warnings', 'experimental::smartmatch'; use ksb::Debug; use ksb::Util; use ksb::BuildContext 0.35; use ksb::BuildSystem::QMake; use ksb::BuildException 0.20; use ksb::Module; use ksb::ModuleResolver 0.20; use ksb::ModuleSet 0.20; use ksb::ModuleSet::KDEProjects; use ksb::RecursiveFH; use ksb::DependencyResolver 0.20; use ksb::Updater::Git; use ksb::Version qw(scriptVersion); use Mojo::IOLoop; use Mojo::Message::Request; use Mojo::JSON qw(encode_json); use Mojo::Promise; use List::Util qw(first min); use File::Basename; # basename, dirname use File::Glob ':glob'; use POSIX qw(:sys_wait_h _exit :errno_h); use Getopt::Long qw(GetOptionsFromArray :config gnu_getopt nobundling); use IO::Handle; use IO::Select; ### Package-specific variables (not shared outside this file). # This is a hash since Perl doesn't have a "in" keyword. my %ignore_list; # List of packages to refuse to include in the build list. use constant { # We use a named remote to make some git commands work that don't accept the # full path. KDE_PROJECT_ID => 'kde-projects', # git-repository-base for kde_projects.xml }; ### Package methods sub new { my ($class, @options) = @_; my $self = bless { context => ksb::BuildContext->new(), metadata_module => undef, run_mode => 'build', modules => undef, module_factory => undef, # ref to sub that makes a new Module. # See generateModuleList _base_pid => $$, # See finish() }, $class; # Default to colorized output if sending to TTY ksb::Debug::setColorfulOutput(-t STDOUT); my @moduleList = $self->generateModuleList(@options); $self->{modules} = \@moduleList; if (!@moduleList) { print "No modules to build, exiting.\n"; exit 0; } $self->context()->setupOperatingEnvironment(); # i.e. niceness, ulimits, etc. # After this call, we must run the finish() method # to cleanly complete process execution. if (!pretending() && !$self->context()->takeLock()) { print "$0 is already running!\n"; exit 1; # Don't finish(), it's not our lockfile!! } # Install signal handlers to ensure that the lockfile gets closed. _installSignalHandlers(sub { note ("Signal received, terminating."); @main::atexit_subs = (); # Remove their finish, doin' it manually $self->finish(5); }); return $self; } # Method: _readCommandLineOptionsAndSelectors # # Returns a list of module/module-set selectors, selected module/module-set # options, and global options, based on the command-line arguments passed to # this function. # # This is a package method, should be called as # $app->_readCommandLineOptionsAndSelectors # # Phase: # initialization - Do not call from this function. # # Parameters: # cmdlineOptions - hashref to hold parsed modules options to be applied later. # *Note* this must be done separately, it is not handled by this subroutine. # Global options will be stored in a hashref at $cmdlineOptions->{global}. # Module or module-set options will be stored in a hashref at # $cmdlineOptions->{$moduleName} (it will be necessary to disambiguate # later in the run whether it is a module set or a single module). # # If the global option 'start-program' is set, then the program to start and # its options will be found in a listref pointed to under the # 'start-program' option. # # selectors - listref to hold the list of module or module-set selectors to # build, in the order desired by the user. These will just be strings, the # caller will have to figure out whether the selector is a module or # module-set, and create any needed objects, and then set the recommended # options as listed in cmdlineOptions. # # ctx - to hold the global build state. # # @options - The remainder of the arguments are treated as command line # arguments to process. # # Returns: # Nothing. An exception will be raised on failure, or this function may quit # the program directly (e.g. to handle --help, --usage). sub _readCommandLineOptionsAndSelectors { my $self = shift; my ($cmdlineOptionsRef, $selectorsRef, $ctx, @options) = @_; my $phases = $ctx->phases(); my @savedOptions = @options; # Copied for use in debugging. my $version = "kdesrc-build " . scriptVersion(); my $author = < Many people have contributed code, bugfixes, and documentation. Please report bugs using the KDE Bugzilla, at https://bugs.kde.org/ DONE # Getopt::Long will store options in %foundOptions, since that is what we # pass in. To allow for custom subroutines to handle an option it is # required that the sub *also* be in %foundOptions... whereupon it will # promptly be overwritten if we're not careful. Instead we let the custom # subs save to %auxOptions, and read those in back over it later. my (%foundOptions, %auxOptions); %foundOptions = ( version => sub { say $version; exit }, author => sub { say $author; exit }, help => sub { _showHelpMessage(); exit 0 }, install => sub { $self->{run_mode} = 'install'; $phases->phases('install'); }, uninstall => sub { $self->{run_mode} = 'uninstall'; $phases->phases('uninstall'); }, 'no-src' => sub { $phases->filterOutPhase('update'); }, 'no-install' => sub { $phases->filterOutPhase('install'); }, 'no-tests' => sub { # The "right thing" to do $phases->filterOutPhase('test'); # What actually works at this point. $foundOptions{'run-tests'} = 0; }, 'no-build' => sub { $phases->filterOutPhase('build'); }, # Mostly equivalent to the above 'src-only' => sub { $phases->phases('update'); # We have an auto-switching function that we only want to run # if --src-only was passed to the command line, so we still # need to set a flag for it. $foundOptions{'allow-auto-repo-move'} = 1; }, 'build-only' => sub { $phases->phases('build'); }, 'install-only' => sub { $self->{run_mode} = 'install'; $phases->phases('install'); }, prefix => sub { my ($optName, $arg) = @_; $auxOptions{prefix} = $arg; $foundOptions{kdedir} = $arg; #TODO: Still needed for compat? $foundOptions{reconfigure} = 1; }, query => sub { my (undef, $arg) = @_; my $validMode = qr/^[a-zA-Z0-9_][a-zA-Z0-9_-]*$/; die("Invalid query mode $arg") unless $arg =~ $validMode; # Add useful aliases $arg = 'source-dir' if $arg =~ /^src-?dir$/; $arg = 'build-dir' if $arg =~ /^build-?dir$/; $arg = 'install-dir' if $arg eq 'prefix'; $self->{run_mode} = 'query'; $auxOptions{query} = $arg; $auxOptions{pretend} = 1; # Implied pretend mode }, pretend => sub { # Set pretend mode but also force the build process to run. $auxOptions{pretend} = 1; $foundOptions{'build-when-unchanged'} = 1; }, resume => sub { $auxOptions{resume} = 1; $phases->filterOutPhase('update'); # Implied --no-src $foundOptions{'no-metadata'} = 1; # Implied --no-metadata }, verbose => sub { $foundOptions{'debug-level'} = ksb::Debug::WHISPER }, quiet => sub { $foundOptions{'debug-level'} = ksb::Debug::NOTE }, 'really-quiet' => sub { $foundOptions{'debug-level'} = ksb::Debug::WARNING }, debug => sub { $foundOptions{'debug-level'} = ksb::Debug::DEBUG; debug ("Commandline was: ", join(', ', @savedOptions)); }, # Hack to set module options 'set-module-option-value' => sub { my ($optName, $arg) = @_; my ($module, $option, $value) = split (',', $arg, 3); if ($module && $option) { $cmdlineOptionsRef->{$module} //= { }; $cmdlineOptionsRef->{$module}->{$option} = $value; } }, # Getopt::Long doesn't set these up for us even though we specify an # array. Set them up ourselves. 'start-program' => [ ], 'ignore-modules' => [ ], # Module selectors, the <> is Getopt::Long shortcut for an # unrecognized non-option value (i.e. an actual argument) '<>' => sub { my $arg = shift; push @{$selectorsRef}, $arg; }, ); # Handle any "cmdline-eligible" options not already covered. my $flagHandler = sub { my ($optName, $optValue) = @_; # Assume to set if nothing provided. $optValue = 1 if (!defined $optValue or $optValue eq ''); $optValue = 0 if lc($optValue) eq 'false'; $optValue = 0 if !$optValue; $auxOptions{$optName} = $optValue; }; foreach my $option (keys %ksb::BuildContext::defaultGlobalFlags) { if (!exists $foundOptions{$option}) { $foundOptions{$option} = $flagHandler; # A ref to a sub here! } } # Actually read the options. my $optsSuccess = GetOptionsFromArray(\@options, \%foundOptions, 'version', 'author', 'help', 'disable-snapshots|no-snapshots', 'install', 'uninstall', 'no-src|no-svn', 'no-install', 'no-build', 'no-tests', 'build-when-unchanged|force-build', 'no-metadata', 'verbose|v', 'quiet|quite|q', 'really-quiet', 'debug', 'reconfigure', 'colorful-output|color!', 'async!', 'src-only|svn-only', 'build-only', 'install-only', 'build-system-only', 'rc-file=s', 'prefix=s', 'niceness|nice:10', 'ignore-modules=s{,}', 'print-modules', 'pretend|dry-run|p', 'refresh-build', 'query=s', 'start-program|run=s{,}', 'revision=i', 'resume-from=s', 'resume-after=s', 'rebuild-failures', 'resume', 'stop-on-failure', 'stop-after=s', 'stop-before=s', 'set-module-option-value=s', 'metadata-only', 'include-dependencies', # Special sub used (see above), but have to tell Getopt::Long to look # for strings (map { "$_:s" } (keys %ksb::BuildContext::defaultGlobalFlags)), # Default handling fine, still have to ask for strings. (map { "$_:s" } (keys %ksb::BuildContext::defaultGlobalOptions)), '<>', # Required to read non-option args ); if (!$optsSuccess) { croak_runtime("Error reading command-line options."); } # To store the values we found, need to strip out the values that are # subroutines, as those are the ones we created. Alternately, place the # subs inline as an argument to the appropriate option in the # GetOptionsFromArray call above, but that's ugly too. my @readOptionNames = grep { ref($foundOptions{$_}) ne 'CODE' } (keys %foundOptions); # Slice assignment: $left{$key} = $right{$key} foreach $key (@keys), but # with hashref syntax everywhere. @{ $cmdlineOptionsRef->{'global'} }{@readOptionNames} = @foundOptions{@readOptionNames}; @{ $cmdlineOptionsRef->{'global'} }{keys %auxOptions} = values %auxOptions; } # Generates the build context and module list based on the command line options # and module selectors provided, and sets up the module factory. # # After this function is called all module set selectors will have been # expanded, and we will know if we need to download kde-projects metadata or # not. Dependency resolution has not occurred. # # Returns: List of Modules to build. sub generateModuleList { my $self = shift; my @argv = @_; # Note: Don't change the order around unless you're sure of what you're # doing. my $ctx = $self->context(); my $cmdlineOptions = { global => { }, }; my $cmdlineGlobalOptions = $cmdlineOptions->{global}; my $deferredOptions = { }; # 'options' blocks # Process --help, --install, etc. first. my @selectors; $self->_readCommandLineOptionsAndSelectors($cmdlineOptions, \@selectors, $ctx, @argv); my %ignoredSelectors; @ignoredSelectors{@{$cmdlineGlobalOptions->{'ignore-modules'}}} = undef; my @startProgramAndArgs = @{$cmdlineGlobalOptions->{'start-program'}}; delete @{$cmdlineGlobalOptions}{qw/ignore-modules start-program/}; # rc-file needs special handling. if (exists $cmdlineGlobalOptions->{'rc-file'} && $cmdlineGlobalOptions->{'rc-file'}) { $ctx->setRcFile($cmdlineGlobalOptions->{'rc-file'}); } # disable async if only running a single phase. $cmdlineGlobalOptions->{async} = 0 if (scalar $ctx->phases()->phases() == 1); my $fh = $ctx->loadRcFile(); $ctx->loadPersistentOptions(); if (exists $cmdlineGlobalOptions->{'resume'}) { my $moduleList = $ctx->getPersistentOption('global', 'resume-list'); if (!$moduleList) { error ("b[--resume] specified, but unable to find resume point!"); error ("Perhaps try b[--resume-from] or b[--resume-after]?"); croak_runtime("Invalid --resume flag"); } unshift @selectors, split(/,\s*/, $moduleList); } if (exists $cmdlineGlobalOptions->{'rebuild-failures'}) { my $moduleList = $ctx->getPersistentOption('global', 'last-failed-module-list'); if (!$moduleList) { error ("b[y[--rebuild-failures] was specified, but unable to determine"); error ("which modules have previously failed to build."); croak_runtime("Invalid --rebuild-failures flag"); } unshift @selectors, split(/,\s*/, $moduleList); } # _readConfigurationOptions will add pending global opts to ctx while ensuring # returned modules/sets have any such options stripped out. It will also add # module-specific options to any returned modules/sets. my @optionModulesAndSets = _readConfigurationOptions($ctx, $fh, $deferredOptions); close $fh; # Check if we're supposed to drop into an interactive shell instead. If so, # here's the stop off point. if (@startProgramAndArgs) { $ctx->setupEnvironment(); # Read options from set-env $ctx->commitEnvironmentChanges(); # Apply env options to environment _executeCommandLineProgram(@startProgramAndArgs); # noreturn } # Everything else in cmdlineOptions should be OK to apply directly as a module # or context option. $ctx->setOption(%{$cmdlineGlobalOptions}); # Selecting modules or module sets would requires having the KDE # build metadata (kde-build-metadata and sysadmin/repo-metadata) # available. $ctx->setKDEDependenciesMetadataModuleNeeded(); $ctx->setKDEProjectsMetadataModuleNeeded(); ksb::Updater::Git::verifyGitConfig(); $self->_downloadKDEProjectMetadata(); # The user might only want metadata to update to allow for a later # --pretend run, check for that here. if (exists $cmdlineGlobalOptions->{'metadata-only'}) { return; } # At this point we have our list of candidate modules / module-sets (as read in # from rc-file). The module sets have not been expanded into modules. # We also might have cmdline "selectors" to determine which modules or # module-sets to choose. First let's select module sets, and expand them. my @globalCmdlineArgs = keys %{$cmdlineGlobalOptions}; my $commandLineModules = scalar @selectors; my $moduleResolver = ksb::ModuleResolver->new($ctx); $moduleResolver->setCmdlineOptions($cmdlineOptions); $moduleResolver->setDeferredOptions($deferredOptions); $moduleResolver->setInputModulesAndOptions(\@optionModulesAndSets); $moduleResolver->setIgnoredSelectors([keys %ignoredSelectors]); $self->_defineNewModuleFactory($moduleResolver); my @modules; if ($commandLineModules) { @modules = $moduleResolver->resolveSelectorsIntoModules(@selectors); ksb::Module->setModuleSource('cmdline'); } else { # Build everything in the rc-file, in the order specified. @modules = $moduleResolver->expandModuleSets(@optionModulesAndSets); if ($ctx->getOption('kde-languages')) { @modules = _expandl10nModules($ctx, @modules); } ksb::Module->setModuleSource('config'); } # Check for ignored modules (post-expansion) @modules = grep { ! exists $ignoredSelectors{$_->name()} } @modules; # If modules were on the command line then they are effectively forced to # process unless overridden by command line options as well. If phases # *were* overridden on the command line, then no update pass is required # (all modules already have correct phases) @modules = _updateModulePhases(@modules) unless $commandLineModules; return @modules; } # Causes kde-projects metadata to be downloaded (unless --pretend, --no-src, or # --no-metadata is in effect, although we'll download even in --pretend if # nothing is available). # # No return value. sub _downloadKDEProjectMetadata { my $self = shift; my $ctx = $self->context(); my $updateStillNeeded = 0; my $wasPretending = pretending(); eval { for my $metadataModule ( $ctx->getKDEDependenciesMetadataModule(), $ctx->getKDEProjectsMetadataModule()) { my $sourceDir = $metadataModule->getSourceDir(); super_mkdir($sourceDir); my $moduleSource = $metadataModule->fullpath('source'); my $updateDesired = !$ctx->getOption('no-metadata') && $ctx->phases()->has('update'); my $updateNeeded = (! -e $moduleSource) || is_dir_empty($moduleSource); my $lastUpdate = $ctx->getPersistentOption('global', 'last-metadata-update') // 0; $updateStillNeeded ||= $updateNeeded; if (!$updateDesired && $updateNeeded && (time - ($lastUpdate)) >= 7200) { warning (" r[b[*] Skipping build metadata update, but it hasn't been updated recently!"); } if ($updateNeeded && pretending()) { warning (" y[b[*] Ignoring y[b[--pretend] option to download required metadata\n" . " y[b[*] --pretend mode will resume after metadata is available."); ksb::Debug::setPretending(0); } if ($updateDesired && (!pretending() || $updateNeeded)) { $metadataModule->scm()->updateInternal(); $ctx->setPersistentOption('global', 'last-metadata-update', time); } ksb::Debug::setPretending($wasPretending); } }; my $err = $@; ksb::Debug::setPretending($wasPretending); if ($err) { die $err if $updateStillNeeded; # Assume previously-updated metadata will work if not updating warning (" b[r[*] Unable to download required metadata for build process"); warning (" b[r[*] Will attempt to press onward..."); warning (" b[r[*] Exception message: $@"); } } # Returns a list of Modules in the proper build order according to the # kde-build-metadata dependency information. # # The kde-build-metadata repository must have already been updated, and the # module factory must be setup. The Modules to reorder must be passed as # arguments. sub _resolveModuleDependencies { my $self = shift; my $ctx = $self->context(); my $metadataModule = $ctx->getKDEDependenciesMetadataModule(); my @modules = @_; @modules = eval { my $dependencyResolver = ksb::DependencyResolver->new($self->{module_factory}); my $branchGroup = $ctx->effectiveBranchGroup(); for my $file ('dependency-data-common', "dependency-data-$branchGroup") { my $dependencyFile = $metadataModule->fullpath('source') . "/$file"; my $dependencies = pretend_open($dependencyFile) or die "Unable to open $dependencyFile: $!"; debug (" -- Reading dependencies from $dependencyFile"); $dependencyResolver->readDependencyData($dependencies); close $dependencies; } my @reorderedModules = $dependencyResolver->resolveDependencies(@modules); return @reorderedModules; }; if ($@) { warning (" r[b[*] Problems encountered trying to sort modules into correct order:"); warning (" r[b[*] $@"); warning (" r[b[*] Will attempt to continue."); } return @modules; } # Runs all update, build, install, etc. phases. Basically this *is* the # script. # The metadata module must already have performed its update by this point. sub runAllModulePhases { my $self = shift; my $ctx = $self->context(); my $metadataModule = $ctx->getKDEDependenciesMetadataModule(); my @modules = $self->modules(); $ctx->addToIgnoreList($metadataModule->scm()->ignoredModules()); # Remove modules that are explicitly blanked out in their branch-group # i.e. those modules where they *have* a branch-group, and it's set to # be empty (""). my $resolver = $ctx->moduleBranchGroupResolver(); my $branchGroup = $ctx->effectiveBranchGroup(); @modules = grep { my $branch = $_->isKDEProject() ? $resolver->findModuleBranch($_->fullProjectPath(), $branchGroup) : 1; # Just a placeholder truthy value whisper ("Removing ", $_->fullProjectPath(), " due to branch-group") if (defined $branch and !$branch); (!defined $branch or $branch); # This is the actual test } (@modules); @modules = $self->_resolveModuleDependencies(@modules); # Filter --resume-foo options. This might be a second pass, but that should # be OK since there's nothing different going on from the first pass (in # resolveSelectorsIntoModules) in that event. @modules = _applyModuleFilters($ctx, @modules); if ($ctx->getOption('print-modules')) { info (" * Module list", $metadataModule ? " in dependency order" : ''); for my $m (@modules) { say ((" " x ($m->getOption('#dependency-level', 'module') // 0)), "$m"); } return 0; # Abort execution early! } # Add to global module list now that we've filtered everything. $ctx->addModule($_) foreach @modules; my $runMode = $self->runMode(); if ($runMode eq 'query') { my $queryMode = $ctx->getOption('query', 'module'); # Default to ->getOption as query method. # $_[0] is short name for first param. my $query = sub { $_[0]->getOption($queryMode) }; $query = sub { $_[0]->fullpath('source') } if $queryMode eq 'source-dir'; $query = sub { $_[0]->fullpath('build') } if $queryMode eq 'build-dir'; $query = sub { $_[0]->installationPath() } if $queryMode eq 'install-dir'; $query = sub { $_[0]->fullProjectPath() } if $queryMode eq 'project-path'; $query = sub { ($_[0]->scm()->_determinePreferredCheckoutSource())[0] // '' } if $queryMode eq 'branch'; if (@modules == 1) { # No leading module name, just the value say $query->($modules[0]); } else { for my $m (@modules) { say "$m: ", $query->($m); } } return 0; } my $result; if ($runMode eq 'build') { # No packages to install, we're in build mode # What we're going to do is fork another child to perform the source # updates while we build. my $updateOptsSub = sub { my ($k, $v) = @_; $ctx->setPersistentOption($k, $v); }; $result = _handle_async_build ($ctx); # $ipc->outputPendingLoggedMessages() if debugging(); } elsif ($runMode eq 'install') { $result = _handle_install ($ctx); } elsif ($runMode eq 'uninstall') { $result = _handle_uninstall ($ctx); } _cleanup_log_directory($ctx) if $ctx->getOption('purge-old-logs'); _output_failed_module_lists($ctx); # Record all failed modules. Unlike the 'resume-list' option this doesn't # include any successfully-built modules in between failures. my $failedModules = join(',', map { "$_" } $ctx->listFailedModules()); if ($failedModules) { # We don't clear the list of failed modules on success so that # someone can build one or two modules and still use # --rebuild-failures $ctx->setPersistentOption('global', 'last-failed-module-list', $failedModules); } # env driver is just the ~/.config/kde-env-*.sh, session driver is that + ~/.xsession if ($ctx->getOption('install-environment-driver') || $ctx->getOption('install-session-driver')) { _installCustomSessionDriver($ctx); } my $color = 'g[b['; $color = 'r[b[' if $result; info ("${color}", $result ? ":-(" : ":-)") unless pretending(); return $result; } # Method: finish # # Exits the script cleanly, including removing any lock files created. # # Parameters: # ctx - Required; BuildContext to use. # [exit] - Optional; if passed, is used as the exit code, otherwise 0 is used. sub finish { my $self = shift; my $ctx = $self->context(); my $exitcode = shift // 0; if (pretending() || $self->{_base_pid} != $$) { # Abort early if pretending or if we're not the same process # that was started by the user (e.g. async mode, forked pipe-opens exit $exitcode; } $ctx->closeLock(); $ctx->storePersistentOptions(); my $logdir = $ctx->getLogDir(); note ("Your logs are saved in y[$logdir]"); exit $exitcode; } ### Package-internal helper functions. # Reads a "line" from a file. This line is stripped of comments and extraneous # whitespace. Also, backslash-continued multiple lines are merged into a single # line. # # First parameter is the reference to the filehandle to read from. # Returns the text of the line. sub _readNextLogicalLine { my $fileReader = shift; while($_ = $fileReader->readLine()) { # Remove trailing newline chomp; # Replace \ followed by optional space at EOL and try again. if(s/\\\s*$//) { $_ .= $fileReader->readLine(); redo; } s/#.*$//; # Remove comments next if /^\s*$/; # Skip blank lines return $_; } return undef; } # Takes an input line, and extracts it into an option name, and simplified # value. The value has "false" converted to 0, white space simplified (like in # Qt), and tildes (~) in what appear to be path-like entries are converted to # the home directory path. # # First parameter is the build context (used for translating option values). # Second parameter is the line to split. # Return value is (option-name, option-value) sub _splitOptionAndValue { my $ctx = assert_isa(shift, 'ksb::BuildContext'); my $input = shift; my $optionRE = qr/\$\{([a-zA-Z0-9-]+)\}/; # The option is the first word, followed by the # flags on the rest of the line. The interpretation # of the flags is dependant on the option. my ($option, $value) = ($input =~ /^\s* # Find all spaces ([-\w]+) # First match, alphanumeric, -, and _ # (?: ) means non-capturing group, so (.*) is $value # So, skip spaces and pick up the rest of the line. (?:\s+(.*))?$/x); $value //= ''; # Simplify whitespace. $value =~ s/\s+$//; $value =~ s/^\s+//; $value =~ s/\s+/ /g; # Check for false keyword and convert it to Perl false. $value = 0 if lc($value) eq 'false'; # Replace reference to global option with their value. # The regex basically just matches ${option-name}. my ($sub_var_name) = ($value =~ $optionRE); while ($sub_var_name) { my $sub_var_value = $ctx->getOption($sub_var_name) || ''; if(!$ctx->hasOption($sub_var_name)) { warning (" *\n * WARNING: $sub_var_name is not set at line y[$.]\n *"); ## TODO: filename is missing } debug ("Substituting \${$sub_var_name} with $sub_var_value"); $value =~ s/\$\{$sub_var_name\}/$sub_var_value/g; # Replace other references as well. Keep this RE up to date with # the other one. ($sub_var_name) = ($value =~ $optionRE); } # Replace tildes with home directory. 1 while ($value =~ s"(^|:|=)~/"$1$ENV{'HOME'}/"); return ($option, $value); } # Ensures that the given ModuleSet has at least a valid repository and # use-modules setting based on the given BuildContext. sub _validateModuleSet { my ($ctx, $moduleSet) = @_; my $name = $moduleSet->name() || 'unnamed'; my $rcSources = _getModuleSources($moduleSet); # re-read option from module set since it may be pre-set my $selectedRepo = $moduleSet->getOption('repository'); if (!$selectedRepo) { error (<getOption('git-repository-base'); if ($selectedRepo ne KDE_PROJECT_ID && not exists $repoSet->{$selectedRepo}) { my $projectID = KDE_PROJECT_ID; my $moduleSetName = $moduleSet->name(); my $moduleSetId = $moduleSetName ? "module-set ($moduleSetName)" : "module-set"; error (<isa('ksb::BuildContext') ? 'global' : $module->isa('ksb::ModuleSet') ? 'module-set' : $module->isa('ksb::Module') ? 'module' : 'options'; # Just look for an end marker if terminator not provided. $endRE //= qr/^end[\w\s]*$/; _markModuleSource($module, $fileReader->currentFilename() . ":$."); # Read in each option while (($_ = _readNextLogicalLine($fileReader)) && ($_ !~ $endRE)) { my $current_file = $fileReader->currentFilename(); # Sanity check, make sure the section is correctly terminated if(/^(module\b|options\b)/) { error ("Invalid configuration file $current_file at line $.\nAdd an 'end $endWord' before " . "starting a new module.\n"); die make_exception('Config', "Invalid file $current_file"); } my ($option, $value) = _splitOptionAndValue($ctx, $_); eval { $module->setOption($option, $value); }; if (my $err = $@) { if (blessed($err) && $err->isa('ksb::BuildException::Config')) { my $msg = "$current_file:$.: " . $err->message(); my $explanation = $err->optionUsageExplanation(); $msg = $msg . "\n" . $explanation if $explanation; $err->setMessage($msg); } die; # re-throw } } return $module; } # Marks the given OptionsBase subclass (i.e. Module or ModuleSet) as being # read in from the given string (filename:line). An OptionsBase can be # tagged under multiple files. sub _markModuleSource { my ($optionsBase, $configSource) = @_; my $key = '#defined-at'; my $sourcesRef = $optionsBase->hasOption($key) ? $optionsBase->getOption($key) : []; push @$sourcesRef, $configSource; $optionsBase->setOption($key, $sourcesRef); } # Returns rcfile sources for given OptionsBase (comma-separated). sub _getModuleSources { my $optionsBase = shift; my $key = '#defined-at'; my $sourcesRef = $optionsBase->getOption($key) || []; return join(', ', @$sourcesRef); } # Reads in a "moduleset". # # First parameter is the build context. # Second parameter is the filehandle to the config file to read from. # Third parameter is the ksb::ModuleSet to use. # # Returns the ksb::ModuleSet passed in with read-in options set, which may need # to be further expanded (see ksb::ModuleSet::convertToModules). sub _parseModuleSetOptions { my ($ctx, $fileReader, $moduleSet) = @_; $moduleSet = _parseModuleOptions($ctx, $fileReader, $moduleSet, qr/^end\s+module(-?set)?$/); if ($moduleSet->getOption('repository') eq KDE_PROJECT_ID && !$moduleSet->isa('ksb::ModuleSet::KDEProjects')) { # Perl-specific note! re-blessing the module set into the right 'class' # You'd probably have to construct an entirely new object and copy the # members over in other languages. bless $moduleSet, 'ksb::ModuleSet::KDEProjects'; } return $moduleSet; } # Function: _readConfigurationOptions # # Reads in the settings from the configuration, passed in as an open # filehandle. # # Phase: # initialization - Do not call from this function. # # Parameters: # ctx - The to update based on the configuration read and # any pending command-line options (see cmdlineGlobalOptions). # # filehandle - The I/O object to read from. Must handle _eof_ and _readline_ # methods (e.g. subclass). # # deferredOptions - An out paramter: a hashref holding the options set by any # 'options' blocks read in by this function. Each key (identified by the name # of the 'options' block) will point to a hashref value holding the options to # apply. # # Returns: # @module - Heterogenous list of and defined in the # configuration file. No module sets will have been expanded out (either # kde-projects or standard sets). # # Throws: # - Config exceptions. sub _readConfigurationOptions { my $ctx = assert_isa(shift, 'ksb::BuildContext'); my $fh = shift; my $deferredOptionsRef = shift; my @module_list; my $rcfile = $ctx->rcFile(); my ($option, %readModules); my $fileReader = ksb::RecursiveFH->new($rcfile); $fileReader->addFile($fh, $rcfile); # Read in global settings while ($_ = $fileReader->readLine()) { s/#.*$//; # Remove comments s/^\s*//; # Remove leading whitespace next if (/^\s*$/); # Skip blank lines # First command in .kdesrc-buildrc should be a global # options declaration, even if none are defined. if (not /^global\s*$/) { error ("Invalid configuration file: $rcfile."); error ("Expecting global settings section at b[r[line $.]!"); die make_exception('Config', 'Missing global section'); } # Now read in each global option. _parseModuleOptions($ctx, $fileReader, $ctx); last; } my $using_default = 1; my %seenModules; # NOTE! *not* module-sets, *just* modules. my %seenModuleSets; # and vice versa -- named sets only though! my %seenModuleSetItems; # To track option override modules. # Now read in module settings while ($_ = $fileReader->readLine()) { s/#.*$//; # Remove comments s/^\s*//; # Remove leading whitespace next if (/^\s*$/); # Skip blank lines # Get modulename (has dash, dots, slashes, or letters/numbers) my ($type, $modulename) = /^(options|module)\s+([-\/\.\w]+)\s*$/; my $newModule; # 'include' directives can change the current file, so check where we're at $rcfile = $fileReader->currentFilename(); # Module-set? if (not $modulename) { my $moduleSetRE = qr/^module-set\s*([-\/\.\w]+)?\s*$/; ($modulename) = m/$moduleSetRE/; # modulename may be blank -- use the regex directly to match if (not /$moduleSetRE/) { error ("Invalid configuration file $rcfile!"); error ("Expecting a start of module section at r[b[line $.]."); die make_exception('Config', 'Ungrouped/Unknown option'); } if ($modulename && exists $seenModuleSets{$modulename}) { error ("Duplicate module-set $modulename at $rcfile:$."); die make_exception('Config', "Duplicate module-set $modulename defined at $rcfile:$."); } if ($modulename && exists $seenModules{$modulename}) { error ("Name $modulename for module-set at $rcfile:$. is already in use on a module"); die make_exception('Config', "Can't re-use name $modulename for module-set defined at $rcfile:$."); } # A moduleset can give us more than one module to add. $newModule = _parseModuleSetOptions($ctx, $fileReader, ksb::ModuleSet->new($ctx, $modulename || "")); # Save 'use-modules' entries so we can see if later module decls # are overriding/overlaying their options. my @moduleSetItems = $newModule->moduleNamesToFind(); @seenModuleSetItems{@moduleSetItems} = ($newModule) x scalar @moduleSetItems; $seenModuleSets{$modulename} = $newModule if $modulename; } # Duplicate module entry? (Note, this must be checked before the check # below for 'options' sets) elsif (exists $seenModules{$modulename} && $type ne 'options') { error ("Duplicate module declaration b[r[$modulename] on line $. of $rcfile"); die make_exception('Config', "Duplicate module $modulename declared at $rcfile:$."); } # Module/module-set options overrides elsif ($type eq 'options') { my $options = _parseModuleOptions($ctx, $fileReader, ksb::OptionsBase->new()); $deferredOptionsRef->{$modulename} = $options->{options}; next; # Don't add to module list } # Must follow 'options' handling elsif (exists $seenModuleSets{$modulename}) { error ("Name $modulename for module at $rcfile:$. is already in use on a module-set"); die make_exception('Config', "Can't re-use name $modulename for module defined at $rcfile:$."); } else { $newModule = _parseModuleOptions($ctx, $fileReader, ksb::Module->new($ctx, $modulename)); $seenModules{$modulename} = $newModule; } push @module_list, $newModule; $using_default = 0; } while (my ($name, $moduleSet) = each %seenModuleSets) { _validateModuleSet($ctx, $moduleSet); } # If the user doesn't ask to build any modules, build a default set. # The good question is what exactly should be built, but oh well. if ($using_default) { warning (" b[y[*] There do not seem to be any modules to build in your configuration."); return (); } return @module_list; } # Exits out of kdesrc-build, executing the user's preferred shell instead. The # difference is that the environment variables should be as set in kdesrc-build # instead of as read from .bashrc and friends. # # You should pass in the options to run the program with as a list. # # Meant to implement the --run command line option. sub _executeCommandLineProgram { my ($program, @args) = @_; if (!$program) { error ("You need to specify a program with the --run option."); exit 1; # Can't use finish here. } if (($< != $>) && ($> == 0)) { error ("kdesrc-build will not run a program as root unless you really are root."); exit 1; } debug ("Executing b[r[$program] ", join(' ', @args)); exit 0 if pretending(); exec $program, @args or do { # If we get to here, that sucks, but don't continue. error ("Error executing $program: $!"); exit 1; }; } # Function: _split_url # # Subroutine to split a url into a protocol and host sub _split_url { my $url = shift; my ($proto, $host) = ($url =~ m|([^:]*)://([^/]*)/|); return ($proto, $host); } # Function: _check_for_ssh_agent # # Checks if we are supposed to use ssh agent by examining the environment, and # if so checks if ssh-agent has a list of identities. If it doesn't, we run # ssh-add (with no arguments) and inform the user. This can be controlled with # the disable-agent-check parameter. # # Parameters: # 1. Build context sub _check_for_ssh_agent { my $ctx = assert_isa(shift, 'ksb::BuildContext'); # Don't bother with all this if the user isn't even using SSH. return 1 if pretending(); my @svnServers = grep { $_->scmType() eq 'svn' } ($ctx->modulesInPhase('update')); my @gitServers = grep { $_->scmType() eq 'git' } ($ctx->modulesInPhase('update')); my @sshServers = grep { my ($proto, $host) = _split_url($_->getOption('svn-server')); # Check if ssh is explicitly used in the proto, or if the host is the # developer main svn. (defined $proto && $proto =~ /ssh/) || (defined $host && $host =~ /^svn\.kde\.org/); } @svnServers; push @sshServers, grep { # Check for git+ssh:// or git@git.kde.org:/path/etc. my $repo = $_->getOption('repository'); ($repo =~ /^git\+ssh:\/\//) || ($repo =~ /^[a-zA-Z0-9_.]+@.*:\//); } @gitServers; return 1 if (not @sshServers) or $ctx->getOption('disable-agent-check'); whisper ("\tChecking for SSH Agent") if (scalar @sshServers); # We're using ssh to download, see if ssh-agent is running. return 1 unless exists $ENV{'SSH_AGENT_PID'}; my $pid = $ENV{'SSH_AGENT_PID'}; # It's supposed to be running, let's see if there exists the program with # that pid (this check is linux-specific at the moment). if (-d "/proc" and not -e "/proc/$pid") { warning ("r[ *] SSH Agent is enabled, but y[doesn't seem to be running]."); warning ("Since SSH is used to download from Subversion you may want to see why"); warning ("SSH Agent is not working, or correct the environment variable settings."); return 0; } # The agent is running, but does it have any keys? We can't be more specific # with this check because we don't know what key is required. my $noKeys = 0; filter_program_output(sub { $noKeys ||= /no identities/ }, 'ssh-add', '-l'); if ($noKeys) { # Use print so user can't inadvertently keep us quiet about this. print ksb::Debug::colorize (<getOption('ssh-identity-file'); push (@commandLine, $identFile) if $identFile; my $result = system (@commandLine); if ($result) # Run this code for both death-by-signal and nonzero return { my $rcfile = $ctx->rcFile(); print "\nUnable to add SSH identity, aborting.\n"; print "If you don't want kdesrc-build to check in the future,\n"; print ksb::Debug::colorize ("Set the g[disable-agent-check] option to g[true] in your $rcfile.\n\n"); return 0; } } return 1; } # Function: _handle_updates # # Subroutine to update a list of modules. Uses a Mojolicious event loop # to run each update in a subprocess to avoid blocking the script. Only # one update process will exist at a given time. # # Parameters: # 1. Build Context, which will be used to determine the module update list. # 2. A hashref mapping module names to Mojo::Promises. This function generates # a promise for each module update, which can be waited on. # # This function accounts for every module in $ctx's update phase. See # module_promises hashref. # # Returns a Mojo::Promise that can be waited on. The Promise should continue # to update every applicable module even if errors occur -- you must still # check for errors even if the promise successfully resolves. sub _handle_updates { my ($ctx, $module_promises, $module_updates_ref) = @_; my $kdesrc = $ctx->getSourceDir(); my @update_list = $ctx->modulesInPhase('update'); $module_promises->{'global'} = Mojo::Promise->new; # No reason to print out the text if we're not doing anything. if (!@update_list) { return $module_promises->{'global'}->resolve(1); } if (not _check_for_ssh_agent($ctx)) { return $module_promises->{'global'}->reject('ssh-failure'); } whisper ("Creating source directory") unless -e $kdesrc; if (! -e $kdesrc && !super_mkdir ($kdesrc)) { error ("Unable to make directory r[$kdesrc]!"); return $module_promises->{'global'}->reject("Could not mkdir $kdesrc"); } # Create promises for each module to update. $module_promises->{"$_"} = Mojo::Promise->new foreach @update_list; # Once at this point, any errors we get should be limited to a module, # which means we can tell the build thread to start. $module_promises->{'global'}->resolve(1); my @steps = map { my $module = $_; # This sub is called *LATER*, by Mojo::IOLoop::Delay->steps # it needs to be in the loop to close over $module # note: implicit return to `map`, using return keyword breaks immediately, # not later. sub { my ($delay, $last_step_successful) = @_; # fed in from a prior IOLoop::Delay step my $end = $delay->begin(0); # This controls when the Delay proceeds Mojo::IOLoop->subprocess( sub { # called in child process, can block $SIG{INT} = sub { POSIX::_exit(EINTR); }; $0 = 'kdesrc-build-updater'; ksb::Debug::setDebugLevel(ksb::Debug::ERROR); return $module->update($ctx); }, sub { # called in this process, with results # in this case the only result is whether there's an error or not my ($subprocess, $err, $updateMsg) = @_; $module_updates_ref->{"$module"} //= { }; my $promise = $module_promises->{"$module"}; if ($err || !$updateMsg) { $module_updates_ref->{"$module"}->{update} = 'failed'; $promise->reject($err || "$module failed to update"); } else { $module_updates_ref->{"$module"}->{update} = 'success'; $promise->resolve($updateMsg); } $end->(!$err && $updateMsg && $last_step_successful); # proceed to next step } ); }; } (@update_list); my $delay = Mojo::IOLoop->delay( sub { my $delay = shift; $delay->pass(1); # kick off process with defined success code }, @steps, # meat of the updates sub { my ($delay, $success) = @_; # all done $delay->reject('update failed') unless $success; $delay->resolve($success) if $success; } ); return $delay; } # Builds the given module. This should only be called if the module # successfully updated and is ready for build. # # Return value is a promise which will eventually resolve to either a # string noting which phase the failure occurred in (if promise rejects) # or the ksb::Module itself (if promise resolves) sub _buildSingleModule { my ($ctx, $module, $startTimeRef) = @_; $$startTimeRef = time; - my $build_promise = Mojo::Promise->new; + my $build_promise = $module->build(); my $fail_count = $module->getPersistentOption('failure-count') // 0; - Mojo::IOLoop->subprocess( - sub { - # called in child process, can block - $SIG{INT} = sub { POSIX::_exit(EINTR); }; - $0 = 'kdesrc-build-builder'; - - $ctx->resetEnvironment(); - $module->setupEnvironment(); - - return $module->build(); - }, - sub { - # called in this process, with results - # in this case the only result is whether there's an error or not - my ($subprocess, $err, $was_successful) = @_; - - $fail_count = $was_successful ? 0 : $fail_count + 1; + # TODO: Move the elapsed timer stuff in here? + my $promise = $build_promise + ->catch(sub { + my $err = shift; + + # build failed + $err //= 'Unknown reason (kdesrc-build bug)'; + error ("\tr[b[$module] failed to build: $err"); + $fail_count++; + })->then(sub { + $fail_count = 0; + })->finally(sub { $module->setPersistentOption('failure-count', $fail_count); + }); - if ($was_successful && !$err) { - $build_promise->resolve($module); - } - else { - error ("Failed to build $module due to $err") if $err; - $build_promise->reject('build'); - } - } - ); - - return $build_promise; + return $promise; } # Returns undef if build should proceed, otherwise a Promise that will resolve # or reject as appropriate sub _checkForEarlyBuildExit { my $ctx = shift; my @modules = $ctx->modulesInPhase('build'); # No reason to print building messages if we're not building. return Mojo::Promise->new->resolve(0) if scalar @modules == 0; # Check for absolutely essential programs now. if (!_checkForEssentialBuildPrograms($ctx) && !exists $ENV{KDESRC_BUILD_IGNORE_MISSING_PROGRAMS}) { error (" r[b[*] Aborting now to save a lot of wasted time."); error (" y[b[*] export KDESRC_BUILD_IGNORE_MISSING_PROGRAMS=1 and re-run (perhaps with --no-src)"); error (" r[b[*] to continue anyways. If this check was in error please report a bug against"); error (" y[b[*] kdesrc-build at https://bugs.kde.org/"); return Mojo::Promise->new->reject('local-setup'); } return undef; } sub _openStatusFileHandle { my $ctx = shift; my $outfile = pretending() ? '/dev/null' : $ctx->getLogDir() . '/build-status'; my $statusFile; open $statusFile, '>', $outfile or do { error (<modulesInPhase('build'); my $result = 0; if (my $promise = _checkForEarlyBuildExit($ctx)) { return $promise; } # Modules in the build phase but not in the update phase were skipped # so we need to create a shell promise to wait upon my @skippedModules = grep { !exists $module_promises->{"$_"} } (@modules); foreach my $skipped (@skippedModules) { my $promise; my $fail_count = $skipped->getPersistentOption('failure-count') // 0; if (!$skipped->getOption('build-when-unchanged') && $fail_count == 0) { # skip the build too, if it didn't fail last time and not overridden by --no-src $promise = Mojo::Promise->new->reject('No changes to source, no need to build'); } else { $promise = Mojo::Promise->new->resolve('update was skipped'); } $module_promises->{"$skipped"} = $promise; } my $num_modules = scalar @modules; my ($statusFile, $outfile) = _openStatusFileHandle($ctx); my $statusViewer = $ctx->statusViewer(); my $everFailed = 0; my $i = 1; $statusViewer->numberModulesTotal(scalar @modules); # This generates a bunch of subs but doesn't call them yet # See Mojo::IOLoop->delay() below for where they get used my @steps = map { my $module = $_; # Implicit return value sub { my $delay = shift; my $end = $delay->begin; my $start_time = time; my $fail_count = $module->getPersistentOption('failure-count') // 0; my $moduleName = $module->name(); $module_updates_ref->{$moduleName} //= { }; if ($everFailed && $module->getOption('stop-on-failure')) { # Wait for update to be done then continue return $module_promises->{"$module"}->then(sub { $i++; $end->(); }); } my $moduleSet = $module->moduleSet()->name(); my $modOutput = $moduleName; if (debugging(ksb::Debug::WHISPER)) { $modOutput .= " (build system " . $module->buildSystemType() . ")" } $moduleSet = " from g[$moduleSet]" if $moduleSet; note ("Building g[$modOutput]$moduleSet ($i/$num_modules)"); $module_promises->{"$module"}->catch(sub { my $reason = shift; error ("\tUnable to update r[$module]: $reason, build canceled."); ++$fail_count; $statusViewer->numberModulesFailed(1 + $statusViewer->numberModulesFailed); return Mojo::Promise->new->reject('failed update'); })->then(sub { my $updateMsg = shift; note ("\tSource update complete for g[$module]: $updateMsg"); # TODO: Split install into a separate phase so that exception # handler knows which phase had the failure return _buildSingleModule($ctx, $module, \$start_time); })->catch(sub { my $failureReason = shift; my $elapsed = prettify_seconds(time - $start_time); $ctx->markModulePhaseFailed('build', $module); print $statusFile "$module: Failed on build after $elapsed.\n"; note ("\tFailed to build $module: $failureReason"); if (!$everFailed) { # No failures yet, mark this as resume point $everFailed = 1; my $moduleList = join(', ', map { "$_" } ($module, @modules)); $ctx->setPersistentOption('global', 'resume-list', $moduleList); } ++$fail_count; $statusViewer->numberModulesFailed(1 + $statusViewer->numberModulesFailed); $module_updates_ref->{$moduleName}->{build} = 'failed'; # Force this promise chain to stay dead return Mojo::Promise->new->reject('build'); })->then(sub { my $built_module = shift; my $elapsed = prettify_seconds(time - $start_time); print $statusFile "$module: Succeeded after $elapsed.\n"; $fail_count = 0; push @build_done, $moduleName; # Make it show up as a success $statusViewer->numberModulesSucceeded(1 + $statusViewer->numberModulesSucceeded); $module_updates_ref->{$moduleName}->{build} = 'success'; })->finally(sub { $i++; note (""); # Blank line $module->setPersistentOption('failure-count', $fail_count); $module_updates_ref->{$moduleName}->{done} = 'true'; $end->(); # Kick off next step }); }; } (@modules); my $build_promise = $module_promises->{'global'}->then( sub { # success $ctx->unsetPersistentOption('global', 'resume-list'); return Mojo::IOLoop->delay(@steps); }, sub { # failed my $reason = shift; error(" Failed to start updates: r[b[$reason]"); return $reason; }, )->then(sub { # after ->delay(@steps) completes above... if ($statusFile) { close $statusFile; # Update the symlink in latest to point to this file. my $logdir = $ctx->getSubdirPath('log-dir'); if (-l "$logdir/latest/build-status") { safe_unlink("$logdir/latest/build-status"); } symlink($outfile, "$logdir/latest/build-status"); } info ("<<< g[PACKAGES SUCCESSFULLY BUILT] >>>") if scalar @build_done > 0; my $successes = scalar @build_done; # TODO: l10n my $mods = $successes == 1 ? 'module' : 'modules'; if (not pretending()) { # Print out results, and output to a file my $kdesrc = $ctx->getSourceDir(); open my $buildList, ">$kdesrc/successfully-built"; foreach my $module (@build_done) { info ("$module") if $successes <= 10; print $buildList "$module\n"; } close $buildList; info ("Built g[$successes] $mods") if $successes > 10; } else { # Just print out the results if ($successes <= 10) { info ('g[', join ("]\ng[", @build_done), ']'); } else { info ("Built g[$successes] $mods") if $successes > 10; } } info (' '); # Space out nicely return $result; }); return $build_promise; } # Launches a server to handle responding to status requests. # # - $module_updates_ref should be a hashref mapping module *names* to status # information, maintained by calling code. # - $done_promise should be a promise that, once resolved, should indicate that # it is time to shut the server down. # # returns a promise that can be waited on until the server is shut down sub _handle_monitoring { my ($module_updates_ref, $done_promise) = @_; # Setup a simple server to respond to requests about kdesrc-build status my $srv_id = Mojo::IOLoop->server( { path => qq(/tmp/kdesrc-build-uds) }, sub { # Called for each new connection accepted by the listening server my ($loop, $stream, $id) = @_; my $req = Mojo::Message::Request->new; $stream->on(error => sub { my ($stream, $err) = @_; whisper("Error involving Mojo stream ID: $err"); $stream->close; }); # emitted for each chunk $stream->on(read => sub { my ($stream, $bytes_read) = @_; $req->parse($bytes_read); return unless $req->is_finished; return if $req->error; my $path = $req->url->path; my $resp = Mojo::Message::Response->new; $resp->code(200); $resp->headers->content_type('application/json'); if ($path->contains('/list')) { $resp->body(Mojo::JSON::encode_json([keys %{$module_updates_ref}])); } elsif ($path->contains('/status')) { my $mod = $path->[1]; # part after /status/ if ($mod && exists $module_updates_ref->{$mod}) { $resp->body(Mojo::JSON::encode_json({$mod => $module_updates_ref->{$mod}})); } else { $resp->code(400); $resp->headers->content_type('text/plain'); $resp->body('You dun goofed'); } } $stream->write($resp->to_string, sub { # callback on finished write my ($stream) = @_; $stream->close_gracefully; }); }); }); my $time_promise = Mojo::Promise->new; # useful for debugging to ensure server is available for at least a few # seconds. # Mojo::IOLoop->timer(10, sub { $time_promise->resolve; }); Mojo::IOLoop->timer(0, sub { $time_promise->resolve; }); my $stop_promise = Mojo::Promise->all($done_promise, $time_promise)->then(sub { Mojo::IOLoop->acceptor($srv_id)->stop; }); return $stop_promise; } # Function: _handle_async_build # # This subroutine special-cases the handling of the update and build phases, by # performing them concurrently using forked processes and non-blocking I/O. # See Mojo::Promise and Mojo::IOLoop::Subprocess # # This procedure will use multiple processes (the main process and separate # processes for each update or build as they occur). # # Parameters: # 1. Build Context to use, from which the module lists will be determined. # # Returns 0 on success, non-zero on failure. sub _handle_async_build { my ($ctx) = @_; my $kdesrc = $ctx->getSourceDir(); my @updateList = $ctx->modulesInPhase('update'); my $result = 0; my $update_done = 0; my $module_promises = { }; my $module_updates_ref = { }; my $stop_everything_p = Mojo::Promise->new; my $updater_delay = _handle_updates ($ctx, $module_promises, $module_updates_ref); my $build_delay = _handle_build ($ctx, $module_promises, $module_updates_ref); my $monitor_p = _handle_monitoring ($module_updates_ref, $stop_everything_p); # If build finishes first, note that later my $mark_update_done_p = $updater_delay->then(sub { $update_done = 1; }); my $build_done_msg_p = $build_delay->then(sub { # This can happen with a build failure under stop-on-failure note("All builds finished, but still waiting for updates") unless $update_done; }); my @all_promises = ($updater_delay, $build_delay, $mark_update_done_p, $build_done_msg_p); my $promise = Mojo::Promise->all(@all_promises)->then(sub { $stop_everything_p->resolve; }); $promise->catch(sub { my $err = shift; say "Caught error $err"; $result = 1; }); Mojo::IOLoop->stop; # Force the wait below to block $monitor_p->then(sub { Mojo::IOLoop->stop; # FIN })->wait; return $result; } # Function: _handle_install # # Handles the installation process. Simply calls 'make install' in the build # directory, though there is also provision for cleaning the build directory # afterwards, or stopping immediately if there is a build failure (normally # every built module is attempted to be installed). # # Parameters: # 1. Build Context, from which the install list is generated. # # Return value is a shell-style success code (0 == success) sub _handle_install { my $ctx = assert_isa(shift, 'ksb::BuildContext'); my @modules = $ctx->modulesInPhase('install'); @modules = grep { $_->buildSystem()->needsInstalled() } (@modules); my $result = 0; for my $module (@modules) { $ctx->resetEnvironment(); $result = $module->install() || $result; if ($result && $module->getOption('stop-on-failure')) { note ("y[Stopping here]."); return 1; # Error } } return $result; } # Function: _handle_uninstall # # Handles the uninstal process. Simply calls 'make uninstall' in the build # directory, while assuming that Qt or CMake actually handles it. # # The order of the modules is often significant, and it may work better to # uninstall modules in reverse order from how they were installed. However this # code does not automatically reverse the order; modules are uninstalled in the # order determined by the build context. # # This function obeys the 'stop-on-failure' option supported by _handle_install. # # Parameters: # 1. Build Context, from which the uninstall list is generated. # # Return value is a shell-style success code (0 == success) sub _handle_uninstall { my $ctx = assert_isa(shift, 'ksb::BuildContext'); my @modules = $ctx->modulesInPhase('uninstall'); @modules = grep { $_->buildSystem()->needsInstalled() } (@modules); my $result = 0; for my $module (@modules) { $ctx->resetEnvironment(); $result = $module->uninstall() || $result; if ($result && $module->getOption('stop-on-failure')) { note ("y[Stopping here]."); return 1; # Error } } return $result; } # Function: _applyModuleFilters # # Applies any module-specific filtering that is necessary after reading command # line and rc-file options. (This is as opposed to phase filters, which leave # each module as-is but change the phases they operate as part of, this # function could remove a module entirely from the build). # # Used for --resume-{from,after} and --stop-{before,after}, but more could be # added in theory. # This subroutine supports --{resume,stop}-* for both modules and module-sets. # # Parameters: # ctx - in use. # @modules - List of or to apply filters on. # # Returns: # list of or with any inclusion/exclusion filters # applied. Do not assume this list will be a strict subset of the input list, # however the order will not change amongst the input modules. sub _applyModuleFilters { my $ctx = assert_isa(shift, 'ksb::BuildContext'); my @moduleList = @_; if (!$ctx->getOption('resume-from') && !$ctx->getOption('resume-after') && !$ctx->getOption('stop-before') && !$ctx->getOption('stop-after')) { debug ("No command-line filter seems to be present."); return @moduleList; } if ($ctx->getOption('resume-from') && $ctx->getOption('resume-after')) { # This one's an error. error (<getOption('stop-before') && $ctx->getOption('stop-after')) { # This one's an error. error (<getOption('resume-from') || $ctx->getOption('resume-after'); my $startIndex = scalar @moduleList; if ($resumePoint) { debug ("Looking for $resumePoint for --resume-* option"); # || 0 is a hack to force Boolean context. my $filterInclusive = $ctx->getOption('resume-from') || 0; my $found = 0; for (my $i = 0; $i < scalar @moduleList; $i++) { my $module = $moduleList[$i]; $found = $module->name() eq $resumePoint; if ($found) { $startIndex = $filterInclusive ? $i : $i + 1; $startIndex = min($startIndex, scalar @moduleList - 1); last; } } } else { $startIndex = 0; } my $stopPoint = $ctx->getOption('stop-before') || $ctx->getOption('stop-after'); my $stopIndex = 0; if ($stopPoint) { debug ("Looking for $stopPoint for --stop-* option"); # || 0 is a hack to force Boolean context. my $filterInclusive = $ctx->getOption('stop-before') || 0; my $found = 0; for (my $i = $startIndex; $i < scalar @moduleList; $i++) { my $module = $moduleList[$i]; $found = $module->name() eq $stopPoint; if ($found) { $stopIndex = $i - ($filterInclusive ? 1 : 0); last; } } } else { $stopIndex = scalar @moduleList - 1; } if ($startIndex > $stopIndex || scalar @moduleList == 0) { # Lost all modules somehow. croak_runtime("Unknown resume -> stop point $resumePoint -> $stopPoint."); } return @moduleList[$startIndex .. $stopIndex]; } # This defines the factory function needed for lower-level code to properly be # able to create ksb::Module objects from just the module name, while still # having the options be properly set and having the module properly tied into a # context. sub _defineNewModuleFactory { my ($self, $resolver) = @_; my $ctx = $self->context(); $self->{module_factory} = sub { # We used to need a special module-set to ignore virtual deps (they # would throw errors if the name did not exist). But, the resolver # handles that fine as well. return $resolver->resolveModuleIfPresent(shift); }; } # This function converts any 'l10n' references on the command line to return a l10n # module with the proper build system, scm type, etc. # # The languages are selected using global/kde-languages (which should be used # exclusively from the configuration file). sub _expandl10nModules { my ($ctx, @modules) = @_; my $l10n = 'l10n-kde4'; assert_isa($ctx, 'ksb::BuildContext'); # Only filter if 'l10n' is actually present in list. my @matches = grep {$_->name() =~ /^(?:$l10n|l10n)$/} @modules; my @langs = split(' ', $ctx->getOption('kde-languages')); return @modules if (!@matches || !@langs); my $l10nModule; for my $match (@matches) { # Remove all instances of l10n. @modules = grep {$_->name() ne $match->name()} @modules; # Save l10n module if user had it in config. We only save the first # one encountered though. $l10nModule //= $match; } # No l10n module? Just create one. $l10nModule //= ksb::Module->new($ctx, $l10n); whisper ("\tAdding languages ", join(';', @langs), " to build."); $l10nModule->setScmType('l10n'); my $scm = $l10nModule->scm(); # Add all required directories to the l10n module. Its buildsystem should # know to skip scripts and templates. $scm->setLanguageDirs(qw/scripts templates/, @langs); $l10nModule->setBuildSystem($scm); push @modules, $l10nModule; return @modules; } # Updates the built-in phase list for all Modules passed into this function in # accordance with the options set by the user. sub _updateModulePhases { whisper ("Filtering out module phases."); for my $module (@_) { if ($module->getOption('manual-update') || $module->getOption('no-svn') || $module->getOption('no-src')) { $module->phases()->clear(); next; } if ($module->getOption('manual-build')) { $module->phases()->filterOutPhase('build'); $module->phases()->filterOutPhase('test'); $module->phases()->filterOutPhase('install'); } $module->phases()->filterOutPhase('install') unless $module->getOption('install-after-build'); $module->phases()->addPhase('test') if $module->getOption('run-tests'); } return @_; } # This subroutine extract the value from options of the form --option=value, # which can also be expressed as --option value. # # The first parameter is the option that the user passed to the cmd line (e.g. # --prefix=/opt/foo). # The second parameter is a reference to the list of command line options. # # The return value is the value of the option (the list of options might be # shorter by 1, copy it if you don't want it to change), or undef if no value # was provided. sub _extractOptionValue { my ($option, $options_ref) = @_; if ($option =~ /=/) { my @value = split(/=/, $option); shift @value; # We don't need the first one, that the --option part. return if (scalar @value == 0); # If we have more than one element left in @value it's because the # option itself has an = in it, make sure it goes back in the answer. return join('=', @value); } return if scalar @{$options_ref} == 0; return shift @{$options_ref}; } # Like _extractOptionValue, but throws an exception if the value is not # actually present, so you don't have to check for it yourself. If you do get a # return value, it will be defined to something. sub _extractOptionValueRequired { my ($option, $options_ref) = @_; my $returnValue = _extractOptionValue($option, $options_ref); if (not defined $returnValue) { croak_runtime("Option $option needs to be set to some value instead of left blank"); } return $returnValue; } # Function: _cleanup_log_directory # # This function removes log directories from old kdesrc-build runs. All log # directories not referenced by $log_dir/latest somehow are made to go away. # # Parameters: # 1. Build context. # # No return value. sub _cleanup_log_directory { my $ctx = assert_isa(shift, 'ksb::BuildContext'); my $logdir = $ctx->getSubdirPath('log-dir'); return 0 if ! -e "$logdir/latest"; # Could happen for error on first run... # This glob relies on the date being in the specific format YYYY-MM-DD-ID my @dirs = bsd_glob("$logdir/????-??-??-??/", GLOB_NOSORT); my @needed = _reachableModuleLogs("$logdir/latest"); # Convert a list to a hash lookup since Perl lacks a "list-has" my %needed_table; @needed_table{@needed} = (1) x @needed; my $length = scalar @dirs - scalar @needed; if ($length > 15) { # Arbitrary man is arbitrary note ("Removing y[b[$length] out of g[b[$#dirs] old log directories (this may take some time)..."); } elsif ($length > 0) { info ("Removing g[b[$length] out of g[b[$#dirs] old log directories..."); } for my $dir (@dirs) { my ($id) = ($dir =~ m/(\d\d\d\d-\d\d-\d\d-\d\d)/); safe_rmtree($dir) unless $needed_table{$id}; } } # Function: _output_failed_module_list # # Print out an error message, and a list of modules that match that error # message. It will also display the log file name if one can be determined. # The message will be displayed all in uppercase, with PACKAGES prepended, so # all you have to do is give a descriptive message of what this list of # packages failed at doing. # # No message is printed out if the list of failed modules is empty, so this # function can be called unconditionally. # # Parameters: # 1. Build Context # 2. Message to print (e.g. 'failed to foo') # 3. List of ksb::Modules that had failed to foo # # No return value. sub _output_failed_module_list { my ($ctx, $message, @fail_list) = @_; assert_isa($ctx, 'ksb::BuildContext'); $message = uc $message; # Be annoying if (@fail_list) { debug ("Message is $message"); debug ("\tfor ", join(', ', @fail_list)); } if (scalar @fail_list > 0) { my $homedir = $ENV{'HOME'}; my $logfile; warning ("\nr[b[<<< PACKAGES $message >>>]"); for my $module (@fail_list) { $logfile = $module->getOption('#error-log-file'); # async updates may cause us not to have a error log file stored. There's only # one place it should be though, take advantage of side-effect of log_command() # to find it. if (not $logfile) { my $logdir = $module->getLogDir() . "/error.log"; $logfile = $logdir if -e $logdir; } $logfile = "No log file" unless $logfile; $logfile =~ s|$homedir|~|; warning ("r[$module]") if pretending(); warning ("r[$module] - g[$logfile]") if not pretending(); } } } # Function: _output_failed_module_lists # # This subroutine reads the list of failed modules for each phase in the build # context and calls _output_failed_module_list for all the module failures. # # Parameters: # 1. Build context # # Return value: # None sub _output_failed_module_lists { my $ctx = assert_isa(shift, 'ksb::BuildContext'); # This list should correspond to the possible phase names (although # it doesn't yet since the old code didn't, TODO) for my $phase ($ctx->phases()->phases()) { my @failures = $ctx->failedModulesInPhase($phase); _output_failed_module_list($ctx, "failed to $phase", @failures); } # See if any modules fail continuously and warn specifically for them. my @super_fail = grep { ($_->getPersistentOption('failure-count') // 0) > 3 } (@{$ctx->moduleList()}); if (@super_fail) { warning ("\nThe following modules have failed to build 3 or more times in a row:"); warning ("\tr[b[$_]") foreach @super_fail; warning ("\nThere is probably a local error causing this kind of consistent failure, it"); warning ("is recommended to verify no issues on the system.\n"); } } # Function: _installTemplatedFile # # This function takes a given file and a build context, and installs it to a # given location while expanding out template entries within the source file. # # The template language is *extremely* simple: <% foo %> is replaced entirely # with the result of $ctx->getOption(foo, 'no-inherit'). If the result # evaluates false for any reason than an exception is thrown. No quoting of # any sort is used in the result, and there is no way to prevent expansion of # something that resembles the template format. # # Multiple template entries on a line will be replaced. # # The destination file will be created if it does not exist. If the file # already exists then an exception will be thrown. # # Error handling: Any errors will result in an exception being thrown. # # Parameters: # 1. Pathname to the source file (use absolute paths) # 2. Pathname to the destination file (use absolute paths) # 3. Build context to use for looking up template values # # Return value: There is no return value. sub _installTemplatedFile { my ($sourcePath, $destinationPath, $ctx) = @_; assert_isa($ctx, 'ksb::BuildContext'); open (my $input, '<', $sourcePath) or croak_runtime("Unable to open template source $sourcePath: $!"); open (my $output, '>', $destinationPath) or croak_runtime("Unable to open template output $destinationPath: $!"); while (!eof ($input)) { my $line = readline($input); if (!defined ($line)) { croak_runtime("Failed to read from $sourcePath at line $.: $!"); unlink($destinationPath); } # Some lines should only be present in the source as they aid with testing. next if $line =~ /kdesrc-build: filter/; $line =~ s { <% \s* # Template bracket and whitespace ([^\s%]+) # Capture variable name \s*%> # remaining whitespace and closing bracket } { $ctx->getOption($1, 'module') // croak_runtime("Invalid variable $1") }gxe; # Replace all matching expressions, use extended regexp w/ # comments, and replacement is Perl code to execute. (print $output $line) or croak_runtime("Unable to write line to $destinationPath at line $.: $!"); } } # Function: _installCustomFile # # This function installs a source file to a destination path, assuming the # source file is a "templated" source file (see also _installTemplatedFile), and # records a digest of the file actually installed. This function will overwrite # a destination if the destination is identical to the last-installed file. # # Error handling: Any errors will result in an exception being thrown. # # Parameters: # 1. Build context to use for looking up template values, # 2. The full path to the source file. # 3. The full path to the destination file (incl. name) # 4. The key name to use for searching/recording installed MD5 digest. # # Return value: There is no return value. sub _installCustomFile { use File::Copy qw(copy); my $ctx = assert_isa(shift, 'ksb::BuildContext'); my ($sourceFilePath, $destFilePath, $md5KeyName) = @_; my $baseName = basename($sourceFilePath); if (-e $destFilePath) { my $existingMD5 = $ctx->getPersistentOption('/digests', $md5KeyName) // ''; if (fileDigestMD5($destFilePath) ne $existingMD5) { if (!$ctx->getOption('#delete-my-settings')) { error ("\tr[*] Installing \"b[$baseName]\" would overwrite an existing file:"); error ("\tr[*] y[b[$destFilePath]"); error ("\tr[*] If this is acceptable, please delete the existing file and re-run,"); error ("\tr[*] or pass b[--delete-my-settings] and re-run."); return; } elsif (!pretending()) { copy ($destFilePath, "$destFilePath.kdesrc-build-backup"); } } } if (!pretending()) { _installTemplatedFile($sourceFilePath, $destFilePath, $ctx); $ctx->setPersistentOption('/digests', $md5KeyName, fileDigestMD5($destFilePath)); } } # Function: _installCustomSessionDriver # # This function installs the included sample .xsession and environment variable # setup files, and records the md5sum of the installed results. # # If a file already exists, then its md5sum is taken and if the same as what # was previously installed, is overwritten. If not the same, the original file # is left in place and the .xsession is instead installed to # .xsession-kdesrc-build # # Error handling: Any errors will result in an exception being thrown. # # Parameters: # 1. Build context to use for looking up template values, # # Return value: There is no return value. sub _installCustomSessionDriver { use FindBin qw($RealBin); use List::Util qw(first); use File::Copy qw(copy); my $ctx = assert_isa(shift, 'ksb::BuildContext'); my @xdgDataDirs = split(':', $ENV{XDG_DATA_DIRS} || '/usr/local/share/:/usr/share/'); my $xdgDataHome = $ENV{XDG_DATA_HOME} || "$ENV{HOME}/.local/share"; # First we have to find the source my @searchPaths = ($RealBin, map { "$_/apps/kdesrc-build" } ($xdgDataHome, @xdgDataDirs)); s{/+$}{} foreach @searchPaths; # Remove trailing slashes s{//+}{/}g foreach @searchPaths; # Remove duplicate slashes my $envScript = first { -f $_ } ( map { "$_/sample-kde-env-master.sh" } @searchPaths ); my $sessionScript = first { -f $_ } ( map { "$_/sample-xsession.sh" } @searchPaths ); if (!$envScript || !$sessionScript) { warning ("b[*] Unable to find helper files to setup a login session."); warning ("b[*] You will have to setup login yourself, or install kdesrc-build properly."); return; } my $destDir = $ENV{XDG_CONFIG_HOME} || "$ENV{HOME}/.config"; super_mkdir($destDir) unless -d $destDir; _installCustomFile($ctx, $envScript, "$destDir/kde-env-master.sh", 'kde-env-master-digest'); _installCustomFile($ctx, $sessionScript, "$ENV{HOME}/.xsession", 'xsession-digest') if $ctx->getOption('install-session-driver'); if (!pretending()) { if ($ctx->getOption('install-session-driver') && !chmod (0744, "$ENV{HOME}/.xsession")) { error ("\tb[r[*] Error making b[~/.xsession] executable: $!"); error ("\tb[r[*] If this file is not executable you may not be able to login!"); }; } } # Function: _checkForEssentialBuildPrograms # # This subroutine checks for programs which are absolutely essential to the # *build* process and returns false if they are not all present. Right now this # just means qmake and cmake (although this depends on what modules are # actually present in the build context). # # Parameters: # 1. Build context # # Return value: # None sub _checkForEssentialBuildPrograms { my $ctx = assert_isa(shift, 'ksb::BuildContext'); return 1 if pretending(); my @buildModules = $ctx->modulesInPhase('build'); my %requiredPrograms; my %modulesRequiringProgram; foreach my $module ($ctx->modulesInPhase('build')) { my @progs = $module->buildSystem()->requiredPrograms(); # Deliberately used @, since requiredPrograms can return a list. @requiredPrograms{@progs} = 1; foreach my $prog (@progs) { $modulesRequiringProgram{$prog} //= { }; $modulesRequiringProgram{$prog}->{$module->name()} = 1; } } my $wasError = 0; for my $prog (keys %requiredPrograms) { my %requiredPackages = ( qmake => 'Qt', cmake => 'CMake', ); my $programPath = absPathToExecutable($prog); # qmake is not necessarily named 'qmake' if (!$programPath && $prog eq 'qmake') { $programPath = ksb::BuildSystem::QMake::absPathToQMake(); } if (!$programPath) { # Don't complain about Qt if we're building it... if ($prog eq 'qmake' && ( grep { $_->buildSystemType() eq 'Qt' } (@buildModules)) || pretending() ) { next; } $wasError = 1; my $reqPackage = $requiredPackages{$prog} || $prog; my @modulesNeeding = keys %{$modulesRequiringProgram{$prog}}; local $, = ', '; # List separator in output error (<<"EOF"); Unable to find r[b[$prog]. This program is absolutely essential for building the modules: y[@modulesNeeding]. Please ensure the development packages for $reqPackage are installed by using your distribution's package manager. You can also see the https://techbase.kde.org/Getting_Started/Build/Distributions page for information specific to your distribution (although watch for outdated information :( ). EOF } } return !$wasError; } # Function: _reachableModuleLogs # # Returns a list of module directory IDs that must be kept due to being # referenced from the "latest" symlink. # # This function may call itself recursively if needed. # # Parameters: # 1. The log directory under which to search for symlinks, including the "/latest" # part of the path. sub _reachableModuleLogs { my $logdir = shift; my @dirs; # A lexicalized var (my $foo) is required in face of recursiveness. opendir(my $fh, $logdir) or croak_runtime("Can't opendir $logdir: $!"); my $dir = readdir($fh); while(defined $dir) { if (-l "$logdir/$dir") { my $link = readlink("$logdir/$dir"); push @dirs, $link; } elsif ($dir !~ /^\.{1,2}$/) { # Skip . and .. directories (this is a great idea, trust me) push @dirs, _reachableModuleLogs("$logdir/$dir"); } $dir = readdir $fh; } closedir $fh; # Extract numeric IDs from directory names. @dirs = map { m/(\d{4}-\d\d-\d\d-\d\d)/ } (@dirs); # Convert to unique list by abusing hash keys. my %tempHash; @tempHash{@dirs} = (); return keys %tempHash; } # Installs the given subroutine as a signal handler for a set of signals which # could kill the program. # # First parameter is a reference to the sub to act as the handler. sub _installSignalHandlers { my $handlerRef = shift; my @signals = qw/HUP INT QUIT ABRT TERM PIPE/; @SIG{@signals} = ($handlerRef) x scalar @signals; } # Shows a help message and version. Does not exit. sub _showHelpMessage { my $scriptVersion = scriptVersion(); print <, and others. The script is distributed under the terms of the GNU General Public License v2, and includes ABSOLUTELY NO WARRANTY!!! Options: --no-src Skip contacting the source server. --no-build Skip the build process. --no-install Don't automatically install after build. --pretend Don't actually take major actions, instead describe what would be done. --src-only Only update the source code (Identical to --no-build at this point). --build-only Build only, don't perform updates or install. --install-only Only install the already compiled code, this is equivalent --install to make install/fast in CMake. Useful for example when we want to clean the install directory but we do not want to re-compile everything. --rc-file= Read configuration from filename instead of default. --resume-from= Skips modules until just before the given package, then operates as normal. --resume-after= Skips modules up to and including the given package, then operates as normal. --stop-before= Skips the given package and all later packages. --stop-after= Skips all packages after the given package. --stop-on-failure Stops the build as soon as a package fails to build. --reconfigure Run CMake/configure again, but don't clean the build directory. --build-system-only Create the build infrastructure, but don't actually perform the build. --