diff --git a/modules/ksb/Application.pm b/modules/ksb/Application.pm index e42d554..44ac5a4 100644 --- a/modules/ksb/Application.pm +++ b/modules/ksb/Application.pm @@ -1,2468 +1,2473 @@ package ksb::Application 0.20; # Class: Application # # Contains the application-layer logic (i.e. creating a build context, reading # options, parsing command-line, etc.) use strict; use warnings; use 5.014; no if $] >= 5.018, 'warnings', 'experimental::smartmatch'; use ksb::Debug 0.30; use ksb::Util; use ksb::BuildContext 0.35; use ksb::BuildSystem::QMake; use ksb::BuildException 0.20; use ksb::FirstRun; use ksb::Module; use ksb::ModuleResolver 0.20; use ksb::ModuleSet 0.20; use ksb::ModuleSet::KDEProjects; use ksb::ModuleSet::Qt; use ksb::OSSupport; use ksb::PromiseChain; use ksb::RecursiveFH; use ksb::DependencyResolver 0.20; use ksb::DebugOrderHints; use ksb::Updater::Git; use ksb::Version qw(scriptVersion); use Mojo::IOLoop; use Mojo::Promise; use Fcntl; # For sysopen use List::Util qw(first min); use File::Basename; # basename, dirname use File::Glob ':glob'; use POSIX qw(:sys_wait_h _exit :errno_h); use Getopt::Long qw(GetOptionsFromArray :config gnu_getopt nobundling); use IO::Handle; use IO::Select; ### Package-specific variables (not shared outside this file). use constant { # We use a named remote to make some git commands work that don't accept the # full path. KDE_PROJECT_ID => 'kde-projects', # git-repository-base for sysadmin/repo-metadata QT_PROJECT_ID => 'qt-projects', # git-repository-base for qt.io Git repo }; ### Package methods sub new { my ($class, @options) = @_; my $self = bless { context => ksb::BuildContext->new(), metadata_module => undef, run_mode => 'build', modules => undef, module_resolver => undef, # ksb::ModuleResolver but see below _base_pid => $$, # See finish() }, $class; # Default to colorized output if sending to TTY ksb::Debug::setColorfulOutput(-t STDOUT); return $self; } # Call after establishContext (to read in config file and do one-time metadata # reading), but before you call startHeadlessBuild. # # Parameter: # # - workload, a hashref containing the following entries: # { # selectedModules: listref with the selected ksb::Modules to build # dependencyInfo: reference to a dependency info object created by # ksb::DependencyResolver # build: a boolean indicating whether to go through with build or not # } sub setModulesToProcess { my ($self, $workLoad) = @_; croak_internal("Expecting workload object!") unless ref $workLoad eq 'HASH'; $self->{modules} = $workLoad->{selectedModules}; $self->{workLoad} = $workLoad; $self->context()->addModule($_) foreach @{$self->{modules}}; # i.e. niceness, ulimits, etc. $self->context()->setupOperatingEnvironment(); } # Sets the application to be non-interactive, intended to make this suitable as # a backend for a Mojolicious-based web server with a separate U/I. sub setHeadless { my $self = shift; $self->{run_mode} = 'headless'; return $self; } # Method: _readCommandLineOptionsAndSelectors # # Returns a list of module/module-set selectors, selected module/module-set # options, and global options, based on the command-line arguments passed to # this function. # # This is a package method, should be called as # $app->_readCommandLineOptionsAndSelectors # # Phase: # initialization - Do not call from this function. # # Parameters: # cmdlineOptions - hashref to hold parsed modules options to be applied later. # *Note* this must be done separately, it is not handled by this subroutine. # Global options will be stored in a hashref at $cmdlineOptions->{global}. # Module or module-set options will be stored in a hashref at # $cmdlineOptions->{$moduleName} (it will be necessary to disambiguate # later in the run whether it is a module set or a single module). # # If the global option 'start-program' is set, then the program to start and # its options will be found in a listref pointed to under the # 'start-program' option. # # selectors - listref to hold the list of module or module-set selectors to # build, in the order desired by the user. These will just be strings, the # caller will have to figure out whether the selector is a module or # module-set, and create any needed objects, and then set the recommended # options as listed in cmdlineOptions. # # ctx - to hold the global build state. # # @options - The remainder of the arguments are treated as command line # arguments to process. # # Returns: # Nothing. An exception will be raised on failure, or this function may quit # the program directly (e.g. to handle --help, --usage). sub _readCommandLineOptionsAndSelectors { my $self = shift; my ($cmdlineOptionsRef, $selectorsRef, $ctx, @options) = @_; my $phases = $ctx->phases(); my @savedOptions = @options; # Copied for use in debugging. my $os = ksb::OSSupport->new; my $version = "kdesrc-build " . scriptVersion(); my $author = < Many people have contributed code, bugfixes, and documentation. Please report bugs using the KDE Bugzilla, at https://bugs.kde.org/ DONE # Getopt::Long will store options in %foundOptions, since that is what we # pass in. To allow for custom subroutines to handle an option it is # required that the sub *also* be in %foundOptions... whereupon it will # promptly be overwritten if we're not careful. Instead we let the custom # subs save to %auxOptions, and read those in back over it later. my (%foundOptions, %auxOptions); %foundOptions = ( 'show-info' => sub { say $version; say "OS: ", $os->vendorID(); exit }, version => sub { say $version; exit }, author => sub { say $author; exit }, help => sub { _showHelpMessage(); exit 0 }, install => sub { $self->{run_mode} = 'install'; $phases->phases('install'); }, uninstall => sub { $self->{run_mode} = 'uninstall'; $phases->phases('uninstall'); }, 'no-src' => sub { $phases->filterOutPhase('update'); }, 'no-install' => sub { $phases->filterOutPhase('install'); }, 'no-snapshots' => sub { # The documented form of disable-snapshots $auxOptions{'disable-snapshots'} = 1; }, 'no-tests' => sub { # The "right thing" to do $phases->filterOutPhase('test'); # What actually works at this point. $foundOptions{'run-tests'} = 0; }, 'no-build' => sub { $phases->filterOutPhase('build'); }, # Mostly equivalent to the above 'src-only' => sub { $phases->phases('update'); # We have an auto-switching function that we only want to run # if --src-only was passed to the command line, so we still # need to set a flag for it. $foundOptions{'allow-auto-repo-move'} = 1; }, 'build-only' => sub { $phases->phases('build'); }, 'install-only' => sub { $self->{run_mode} = 'install'; $phases->phases('install'); }, prefix => sub { my ($optName, $arg) = @_; $auxOptions{prefix} = $arg; $foundOptions{kdedir} = $arg; #TODO: Still needed for compat? $foundOptions{reconfigure} = 1; }, query => sub { my (undef, $arg) = @_; my $validMode = qr/^[a-zA-Z0-9_][a-zA-Z0-9_-]*$/; die("Invalid query mode $arg") unless $arg =~ $validMode; # Add useful aliases $arg = 'source-dir' if $arg =~ /^src-?dir$/; $arg = 'build-dir' if $arg =~ /^build-?dir$/; $arg = 'install-dir' if $arg eq 'prefix'; $self->{run_mode} = 'query'; $auxOptions{query} = $arg; $auxOptions{pretend} = 1; # Implied pretend mode }, pretend => sub { # Set pretend mode but also force the build process to run. $auxOptions{pretend} = 1; $foundOptions{'build-when-unchanged'} = 1; }, resume => sub { $auxOptions{resume} = 1; $phases->filterOutPhase('update'); # Implied --no-src $foundOptions{'no-metadata'} = 1; # Implied --no-metadata }, verbose => sub { $foundOptions{'debug-level'} = ksb::Debug::WHISPER }, quiet => sub { $foundOptions{'debug-level'} = ksb::Debug::NOTE }, 'really-quiet' => sub { $foundOptions{'debug-level'} = ksb::Debug::WARNING }, debug => sub { $foundOptions{'debug-level'} = ksb::Debug::DEBUG; debug ("Commandline was: ", join(', ', @savedOptions)); }, # Hack to set module options 'set-module-option-value' => sub { my ($optName, $arg) = @_; my ($module, $option, $value) = split (',', $arg, 3); if ($module && $option) { $cmdlineOptionsRef->{$module} //= { }; $cmdlineOptionsRef->{$module}->{$option} = $value; } }, # Getopt::Long doesn't set these up for us even though we specify an # array. Set them up ourselves. 'start-program' => [ ], 'ignore-modules' => [ ], # Module selectors, the <> is Getopt::Long shortcut for an # unrecognized non-option value (i.e. an actual argument) '<>' => sub { my $arg = shift; push @{$selectorsRef}, $arg; }, ); # Handle any "cmdline-eligible" options not already covered. my $flagHandler = sub { my ($optName, $optValue) = @_; # Assume to set if nothing provided. $optValue = 1 if (!defined $optValue or $optValue eq ''); $optValue = 0 if lc($optValue) eq 'false'; $optValue = 0 if !$optValue; $auxOptions{$optName} = $optValue; }; foreach my $option (keys %ksb::BuildContext::defaultGlobalFlags) { if (!exists $foundOptions{$option}) { $foundOptions{$option} = $flagHandler; # A ref to a sub here! } } # Actually read the options. my $optsSuccess = GetOptionsFromArray(\@options, \%foundOptions, # Options here should not duplicate the flags and options defined below # from ksb::BuildContext! 'version|v', 'author', 'help', 'show-info', 'install', 'uninstall', 'no-src|no-svn', 'no-install', 'no-build', 'no-tests', 'build-when-unchanged|force-build', 'no-metadata', 'verbose', 'quiet|quite|q', 'really-quiet', 'debug', 'reconfigure', 'colorful-output|color!', 'src-only|svn-only', 'build-only', 'install-only', 'build-system-only', 'rc-file=s', 'prefix=s', 'niceness|nice:10', 'ignore-modules=s{,}', 'print-modules', 'pretend|dry-run|p', 'refresh-build', 'query=s', 'start-program|run=s{,}', 'launch-browser', 'revision=i', 'resume-from=s', 'resume-after=s', 'rebuild-failures', 'resume', 'stop-after=s', 'stop-before=s', 'set-module-option-value=s', 'metadata-only', 'list-build', 'dependency-tree', # Special sub used (see above), but have to tell Getopt::Long to look # for negatable boolean flags (map { "$_!" } (keys %ksb::BuildContext::defaultGlobalFlags)), # Default handling fine, still have to ask for strings. (map { "$_:s" } (keys %ksb::BuildContext::defaultGlobalOptions)), '<>', # Required to read non-option args ); if (!$optsSuccess) { croak_runtime("Error reading command-line options."); } # To store the values we found, need to strip out the values that are # subroutines, as those are the ones we created. Alternately, place the # subs inline as an argument to the appropriate option in the # GetOptionsFromArray call above, but that's ugly too. my @readOptionNames = grep { ref($foundOptions{$_}) ne 'CODE' } (keys %foundOptions); # Slice assignment: $left{$key} = $right{$key} foreach $key (@keys), but # with hashref syntax everywhere. @{ $cmdlineOptionsRef->{'global'} }{@readOptionNames} = @foundOptions{@readOptionNames}; @{ $cmdlineOptionsRef->{'global'} }{keys %auxOptions} = values %auxOptions; } sub _yieldModuleDependencyTreeEntry { my ($nodeInfo, $module, $context) = @_; my $depth = $nodeInfo->{depth}; my $index = $nodeInfo->{idx}; my $count = $nodeInfo->{count}; my $build = $nodeInfo->{build}; my $currentItem = $nodeInfo->{currentItem}; my $currentBranch = $nodeInfo->{currentBranch}; my $parentItem = $nodeInfo->{parentItem}; my $parentBranch = $nodeInfo->{parentBranch}; my $buildStatus = $build ? 'built' : 'not built'; my $statusInfo = $currentBranch ? "($buildStatus: $currentBranch)" : "($buildStatus)"; my $connectorStack = $context->{stack}; my $prefix = pop(@$connectorStack); while($context->{depth} > $depth) { $prefix = pop(@$connectorStack); --($context->{depth}); } push(@$connectorStack, $prefix); my $connector; if ($depth == 0) { $connector = $prefix . ' ── '; push(@$connectorStack, $prefix . (' ' x 4)); } else { $connector = $prefix . ($index == $count ? '└── ': '├── '); push(@$connectorStack, $prefix . ($index == $count ? ' ' x 4: '│ ')); } $context->{depth} = $depth + 1; $context->{report}($connector . $currentItem . ' ' . $statusInfo); } # Generates the build context, builds various module, dependency and branch # group resolvers, and splits up the provided option/selector mix read from # cmdline into selectors (returned to caller, if any) and pre-built context and # resolvers. # # Use "modulesFromSelectors" to further generate the list of ksb::Modules in # dependency order. # # After this function is called all module set selectors will have been # expanded, and we will have downloaded kde-projects metadata. # # Returns: List of Selectors to build. sub establishContext { my $self = shift; my @argv = @_; # Note: Don't change the order around unless you're sure of what you're # doing. my $ctx = $self->context(); my $cmdlineOptions = { global => { }, }; my $cmdlineGlobalOptions = $cmdlineOptions->{global}; my $deferredOptions = { }; # 'options' blocks # Process --help, --install, etc. first. my @selectors; $self->_readCommandLineOptionsAndSelectors($cmdlineOptions, \@selectors, $ctx, @argv); # Convert list to hash for lookup my %ignoredSelectors = map { $_, 1 } @{$cmdlineGlobalOptions->{'ignore-modules'}}; # Set aside debug-related and other short-circuit cmdline options # for kdesrc-build CLI driver to handle my @debugFlags = qw(dependency-tree list-build metadata-only); $self->{debugFlags} = { map { ($_, 1) } grep { defined $cmdlineGlobalOptions->{$_} } (@debugFlags) }; my @startProgramAndArgs = @{$cmdlineGlobalOptions->{'start-program'}}; delete @{$cmdlineGlobalOptions}{qw/ignore-modules start-program/}; # rc-file needs special handling. if (exists $cmdlineGlobalOptions->{'rc-file'} && $cmdlineGlobalOptions->{'rc-file'}) { $ctx->setRcFile($cmdlineGlobalOptions->{'rc-file'}); } my $fh = $ctx->loadRcFile(); $ctx->loadPersistentOptions(); if (exists $cmdlineGlobalOptions->{'resume'}) { my $moduleList = $ctx->getPersistentOption('global', 'resume-list'); if (!$moduleList) { error ("b[--resume] specified, but unable to find resume point!"); error ("Perhaps try b[--resume-from] or b[--resume-after]?"); croak_runtime("Invalid --resume flag"); } unshift @selectors, split(/,\s*/, $moduleList); } if (exists $cmdlineGlobalOptions->{'rebuild-failures'}) { my $moduleList = $ctx->getPersistentOption('global', 'last-failed-module-list'); if (!$moduleList) { error ("b[y[--rebuild-failures] was specified, but unable to determine"); error ("which modules have previously failed to build."); croak_runtime("Invalid --rebuild-failures flag"); } unshift @selectors, split(/,\s*/, $moduleList); } # _readConfigurationOptions will add pending global opts to ctx while ensuring # returned modules/sets have any such options stripped out. It will also add # module-specific options to any returned modules/sets. my @optionModulesAndSets = _readConfigurationOptions($ctx, $fh, $deferredOptions); close $fh; # Check if we're supposed to drop into an interactive shell instead. If so, # here's the stop off point. if (@startProgramAndArgs) { $ctx->setupEnvironment(); # Read options from set-env $ctx->commitEnvironmentChanges(); # Apply env options to environment _executeCommandLineProgram(@startProgramAndArgs); # noreturn } # Everything else in cmdlineOptions should be OK to apply directly as a module # or context option. $ctx->setOption(%{$cmdlineGlobalOptions}); # Selecting modules or module sets would requires having the KDE # build metadata (kde-build-metadata and sysadmin/repo-metadata) # available. $ctx->setKDEDependenciesMetadataModuleNeeded(); $ctx->setKDEProjectsMetadataModuleNeeded(); if (!exists $ENV{HARNESS_ACTIVE}) { # Running in a test harness, avoid downloading metadata which will be # ignored in the test or making changes to git config ksb::Updater::Git::verifyGitConfig(); $self->_downloadKDEProjectMetadata(); } # At this point we have our list of candidate modules / module-sets (as read in # from rc-file). The module sets have not been expanded into modules. # We also might have cmdline "selectors" to determine which modules or # module-sets to choose. First let's select module sets, and expand them. my $moduleResolver = $self->{module_resolver} = ksb::ModuleResolver->new($ctx); $moduleResolver->setCmdlineOptions($cmdlineOptions); $moduleResolver->setDeferredOptions($deferredOptions); $moduleResolver->setInputModulesAndOptions(\@optionModulesAndSets); $moduleResolver->setIgnoredSelectors([keys %ignoredSelectors]); # The user might only want metadata to update to allow for a later # --pretend run, check for that here. if (exists $cmdlineGlobalOptions->{'metadata-only'}) { return; } return @selectors; } # Requires establishContext to have been called first. Converts string-based # "selectors" for modules or module-sets into a list of ksb::Modules (only # modules, no sets), and returns associated metadata including dependencies. # # After this function is called all module set selectors will have been # expanded, and we will have downloaded kde-projects metadata. # # The modules returned must still be added (using setModulesToProcess) to the # context if you intend to build. This is a separate step to allow for some # introspection prior to making choice to build. # # Returns: A hashref to a workload object (as described in setModulesToProcess) sub modulesFromSelectors { my ($self, @selectors) = @_; my $moduleResolver = $self->{module_resolver}; my $ctx = $self->context(); my @modules; if (@selectors) { @modules = $moduleResolver->resolveSelectorsIntoModules(@selectors); } else { # Build everything in the rc-file, in the order specified. my @rcfileModules = @{$moduleResolver->{inputModulesAndOptions}}; @modules = $moduleResolver->expandModuleSets(@rcfileModules); } # If modules were on the command line then they are effectively forced to # process unless overridden by command line options as well. If phases # *were* overridden on the command line, then no update pass is required # (all modules already have correct phases) @modules = _updateModulePhases(@modules) unless @selectors; # TODO: Verify this does anything still my $metadataModule = $ctx->getKDEDependenciesMetadataModule(); $ctx->addToIgnoreList($metadataModule->scm()->ignoredModules()); # Remove modules that are explicitly blanked out in their branch-group # i.e. those modules where they *have* a branch-group, and it's set to # be empty (""). my $resolver = $ctx->moduleBranchGroupResolver(); my $branchGroup = $ctx->effectiveBranchGroup(); @modules = grep { my $branch = $_->isKDEProject() ? $resolver->findModuleBranch($_->fullProjectPath(), $branchGroup) : 1; # Just a placeholder truthy value whisper ("Removing ", $_->fullProjectPath(), " due to branch-group") if (defined $branch and !$branch); (!defined $branch or $branch); # This is the actual test } (@modules); + my @modulesFromCommand = @modules; + my $moduleGraph = $self->_resolveModuleDependencyGraph(@modules); if (!$moduleGraph || !exists $moduleGraph->{graph}) { croak_runtime("Failed to resolve dependency graph"); } # TODO: Implement --dependency-tree if (exists $self->{debugFlags}->{'dependency-tree'}) { # Save for later introspection $self->{debugFlags}->{'dependency-tree'} = $moduleGraph->{graph}; my $result = { dependencyInfo => $moduleGraph, + modulesFromCommand => \@modulesFromCommand, selectedModules => [], build => 0 }; return $result; } @modules = ksb::DependencyResolver::sortModulesIntoBuildOrder( $moduleGraph->{graph} ); # Filter --resume-foo options. This might be a second pass, but that should # be OK since there's nothing different going on from the first pass (in # resolveSelectorsIntoModules) in that event. @modules = _applyModuleFilters($ctx, @modules); # TODO: Implement 'list-build' option if(exists $self->{debugFlags}->{'list-build'}) { my $result = { dependencyInfo => $moduleGraph, + modulesFromCommand => \@modulesFromCommand, selectedModules => [], build => 0 }; return $result; } my $result = { dependencyInfo => $moduleGraph, + modulesFromCommand => \@modulesFromCommand, selectedModules => \@modules, build => 1 }; return $result; } # Causes kde-projects metadata to be downloaded (unless --pretend, --no-src, or # --no-metadata is in effect, although we'll download even in --pretend if # nothing is available). # # No return value. sub _downloadKDEProjectMetadata { my $self = shift; my $ctx = $self->context(); my $updateStillNeeded = 0; my $wasPretending = pretending(); eval { for my $metadataModule ( $ctx->getKDEDependenciesMetadataModule(), $ctx->getKDEProjectsMetadataModule()) { my $sourceDir = $metadataModule->getSourceDir(); super_mkdir($sourceDir); my $moduleSource = $metadataModule->fullpath('source'); my $updateDesired = !$ctx->getOption('no-metadata') && $ctx->phases()->has('update'); my $updateNeeded = (! -e $moduleSource) || is_dir_empty($moduleSource); my $lastUpdate = $ctx->getPersistentOption('global', 'last-metadata-update') // 0; $updateStillNeeded ||= $updateNeeded; if (!$updateDesired && $updateNeeded && (time - ($lastUpdate)) >= 7200) { warning (" r[b[*] Skipping build metadata update, but it hasn't been updated recently!"); } if ($updateNeeded && pretending()) { warning (" y[b[*] Ignoring y[b[--pretend] option to download required metadata\n" . " y[b[*] --pretend mode will resume after metadata is available."); ksb::Debug::setPretending(0); } if ($updateDesired && (!pretending() || $updateNeeded)) { $metadataModule->scm()->updateInternal(); $ctx->setPersistentOption('global', 'last-metadata-update', time); } ksb::Debug::setPretending($wasPretending); } }; my $err = $@; ksb::Debug::setPretending($wasPretending); if ($err) { die $err if $updateStillNeeded; # Assume previously-updated metadata will work if not updating warning (" b[r[*] Unable to download required metadata for build process"); warning (" b[r[*] Will attempt to press onward..."); warning (" b[r[*] Exception message: $@"); } } # Returns a graph of Modules according to the kde-build-metadata dependency # information. # # The kde-build-metadata repository must have already been updated, and the # module factory must be setup. The modules for which to calculate the graph # must be passed in as arguments sub _resolveModuleDependencyGraph { my $self = shift; my $ctx = $self->context(); my $metadataModule = $ctx->getKDEDependenciesMetadataModule(); my @modules = @_; my $graph = eval { my $moduleResolver = $self->{module_resolver}; my $dependencyResolver = ksb::DependencyResolver->new(sub { # Maps module names (what dep resolver has) to built ksb::Modules # (which we need), needs to include all option handling (cmdline, # rc-file, module-sets, etc) return $moduleResolver->resolveModuleIfPresent(shift); }); my $branchGroup = $ctx->effectiveBranchGroup(); for my $file ('dependency-data-common', "dependency-data-$branchGroup") { my $dependencyFile = $metadataModule->fullpath('source') . "/$file"; my $dependencies = pretend_open($dependencyFile) or die "Unable to open $dependencyFile: $!"; debug (" -- Reading dependencies from $dependencyFile"); $dependencyResolver->readDependencyData($dependencies); close $dependencies; } return $dependencyResolver->resolveToModuleGraph(@modules); }; if ($@) { warning (" r[b[*] Problems encountered trying to determing correct module graph:"); warning (" r[b[*] $@"); warning (" r[b[*] Will attempt to continue."); $graph = { graph => undef, syntaxErrors => 0, cycles => 0, trivialCycles => 0, pathErrors => 0, branchErrors => 0, exception => $@ }; } else { if (!$graph->{graph}) { warning (" r[b[*] Unable to determine correct module graph"); warning (" r[b[*] Will attempt to continue."); } } $graph->{exception} = undef; return $graph; } # Similar to the old interactive runAllModulePhases. Actually performs the # build for the modules selected by setModulesToProcess. # # Returns a Mojo::Promise that must be waited on. The promise resolves to # return a single success/failure result; use the event handler for now to get # more detail during a build. sub startHeadlessBuild { my $self = shift; my $ctx = $self->context(); $ctx->statusMonitor()->createBuildPlan($ctx); my $promiseChain = ksb::PromiseChain->new; my $startPromise = Mojo::Promise->new; # These succeed or die outright $startPromise = _handle_updates ($ctx, $promiseChain, $startPromise); $startPromise = _handle_build ($ctx, $promiseChain, $startPromise); die "Can't obtain build lock" unless $ctx->takeLock(); # Install signal handlers to ensure that the lockfile gets closed. _installSignalHandlers(sub { @main::atexit_subs = (); # Remove their finish, doin' it manually $self->finish(5); }); $startPromise->resolve; # allow build to start once control returned to evt loop my $promise = $promiseChain->makePromiseChain($startPromise)->finally(sub { my @results = @_; my $result = 0; # success, non-zero is failure # Must use ! here to make '0 but true' hack work $result = 1 if defined first { !($_->[0] // 1) } @results; $ctx->statusMonitor()->markBuildDone(); $ctx->closeLock(); my $failedModules = join(',', map { "$_" } $ctx->listFailedModules()); if ($failedModules) { # We don't clear the list of failed modules on success so that # someone can build one or two modules and still use # --rebuild-failures $ctx->setPersistentOption('global', 'last-failed-module-list', $failedModules); } # TODO: Anything to do with this info at this point? my $workLoad = $self->workLoad(); my $dependencyGraph = $workLoad->{dependencyInfo}->{graph}; $ctx->storePersistentOptions(); _cleanup_log_directory($ctx); return $result; }); return $promise; } # Method: finish # # Exits the script cleanly, including removing any lock files created. # # Parameters: # ctx - Required; BuildContext to use. # [exit] - Optional; if passed, is used as the exit code, otherwise 0 is used. sub finish { my $self = shift; my $ctx = $self->context(); my $exitcode = shift // 0; if (pretending() || $self->{_base_pid} != $$) { # Abort early if pretending or if we're not the same process # that was started by the user (for async mode) exit $exitcode; } $ctx->closeLock(); $ctx->storePersistentOptions(); my $logdir = $ctx->getLogDir(); note ("Your logs are saved in y[$logdir]"); exit $exitcode; } ### Package-internal helper functions. # Reads a "line" from a file. This line is stripped of comments and extraneous # whitespace. Also, backslash-continued multiple lines are merged into a single # line. # # First parameter is the reference to the filehandle to read from. # Returns the text of the line. sub _readNextLogicalLine { my $fileReader = shift; while($_ = $fileReader->readLine()) { # Remove trailing newline chomp; # Replace \ followed by optional space at EOL and try again. if(s/\\\s*$//) { $_ .= $fileReader->readLine(); redo; } s/#.*$//; # Remove comments next if /^\s*$/; # Skip blank lines return $_; } return undef; } # Takes an input line, and extracts it into an option name, and simplified # value. The value has "false" converted to 0, white space simplified (like in # Qt), and tildes (~) in what appear to be path-like entries are converted to # the home directory path. # # First parameter is the build context (used for translating option values). # Second parameter is the line to split. # Return value is (option-name, option-value) sub _splitOptionAndValue { my $ctx = assert_isa(shift, 'ksb::BuildContext'); my $input = shift; my $optionRE = qr/\$\{([a-zA-Z0-9-]+)\}/; # The option is the first word, followed by the # flags on the rest of the line. The interpretation # of the flags is dependent on the option. my ($option, $value) = ($input =~ /^\s* # Find all spaces ([-\w]+) # First match, alphanumeric, -, and _ # (?: ) means non-capturing group, so (.*) is $value # So, skip spaces and pick up the rest of the line. (?:\s+(.*))?$/x); $value = trimmed($value // ''); # Simplify whitespace. $value =~ s/\s+/ /g; # Check for false keyword and convert it to Perl false. $value = 0 if lc($value) eq 'false'; # Replace reference to global option with their value. # The regex basically just matches ${option-name}. my ($sub_var_name) = ($value =~ $optionRE); while ($sub_var_name) { my $sub_var_value = $ctx->getOption($sub_var_name) || ''; if(!$ctx->hasOption($sub_var_name)) { warning (" *\n * WARNING: $sub_var_name is not set at line y[$.]\n *"); ## TODO: filename is missing } debug ("Substituting \${$sub_var_name} with $sub_var_value"); $value =~ s/\$\{$sub_var_name\}/$sub_var_value/g; # Replace other references as well. Keep this RE up to date with # the other one. ($sub_var_name) = ($value =~ $optionRE); } # Replace tildes with home directory. 1 while ($value =~ s"(^|:|=)~/"$1$ENV{'HOME'}/"); return ($option, $value); } # Ensures that the given ModuleSet has at least a valid repository and # use-modules setting based on the given BuildContext. sub _validateModuleSet { my ($ctx, $moduleSet) = @_; my $name = $moduleSet->name() || 'unnamed'; my $rcSources = _getModuleSources($moduleSet); # re-read option from module set since it may be pre-set my $selectedRepo = $moduleSet->getOption('repository'); if (!$selectedRepo) { error (<getOption('git-repository-base'); if ($selectedRepo ne KDE_PROJECT_ID && $selectedRepo ne QT_PROJECT_ID && not exists $repoSet->{$selectedRepo}) { my $projectID = KDE_PROJECT_ID; my $moduleSetName = $moduleSet->name(); my $moduleSetId = $moduleSetName ? "module-set ($moduleSetName)" : "module-set"; error (<isa('ksb::BuildContext') ? 'global' : $module->isa('ksb::ModuleSet') ? 'module-set' : $module->isa('ksb::Module') ? 'module' : 'options'; # Just look for an end marker if terminator not provided. $endRE //= qr/^end[\w\s]*$/; _markModuleSource($module, $fileReader->currentFilename() . ":$."); # Read in each option while (($_ = _readNextLogicalLine($fileReader)) && ($_ !~ $endRE)) { my $current_file = $fileReader->currentFilename(); # Sanity check, make sure the section is correctly terminated if(/^(module\b|options\b)/) { error ("Invalid configuration file $current_file at line $.\nAdd an 'end $endWord' before " . "starting a new module.\n"); die make_exception('Config', "Invalid file $current_file"); } my ($option, $value) = _splitOptionAndValue($ctx, $_); eval { $module->setOption($option, $value); }; if (my $err = $@) { if (blessed($err) && $err->isa('ksb::BuildException::Config')) { my $msg = "$current_file:$.: " . $err->message(); my $explanation = $err->optionUsageExplanation(); $msg = $msg . "\n" . $explanation if $explanation; $err->setMessage($msg); } die; # re-throw } } return $module; } # Marks the given OptionsBase subclass (i.e. Module or ModuleSet) as being # read in from the given string (filename:line). An OptionsBase can be # tagged under multiple files. sub _markModuleSource { my ($optionsBase, $configSource) = @_; my $key = '#defined-at'; my $sourcesRef = $optionsBase->hasOption($key) ? $optionsBase->getOption($key) : []; push @$sourcesRef, $configSource; $optionsBase->setOption($key, $sourcesRef); } # Returns rcfile sources for given OptionsBase (comma-separated). sub _getModuleSources { my $optionsBase = shift; my $key = '#defined-at'; my $sourcesRef = $optionsBase->getOption($key) || []; return join(', ', @$sourcesRef); } # Reads in a "moduleset". # # First parameter is the build context. # Second parameter is the filehandle to the config file to read from. # Third parameter is the ksb::ModuleSet to use. # # Returns the ksb::ModuleSet passed in with read-in options set, which may need # to be further expanded (see ksb::ModuleSet::convertToModules). sub _parseModuleSetOptions { my ($ctx, $fileReader, $moduleSet) = @_; $moduleSet = _parseModuleOptions($ctx, $fileReader, $moduleSet, qr/^end\s+module(-?set)?$/); # Perl-specific note! re-blessing the module set into the right 'class' # You'd probably have to construct an entirely new object and copy the # members over in other languages. if ($moduleSet->getOption('repository') eq KDE_PROJECT_ID) { bless $moduleSet, 'ksb::ModuleSet::KDEProjects'; } elsif ($moduleSet->getOption('repository') eq QT_PROJECT_ID) { bless $moduleSet, 'ksb::ModuleSet::Qt'; } return $moduleSet; } # Function: _readConfigurationOptions # # Reads in the settings from the configuration, passed in as an open # filehandle. # # Phase: # initialization - Do not call from this function. # # Parameters: # ctx - The to update based on the configuration read and # any pending command-line options (see cmdlineGlobalOptions). # # filehandle - The I/O object to read from. Must handle _eof_ and _readline_ # methods (e.g. subclass). # # deferredOptions - An out parameter: a hashref holding the options set by any # 'options' blocks read in by this function. Each key (identified by the name # of the 'options' block) will point to a hashref value holding the options to # apply. # # Returns: # @module - Heterogeneous list of and defined in the # configuration file. No module sets will have been expanded out (either # kde-projects or standard sets). # # Throws: # - Config exceptions. sub _readConfigurationOptions { my $ctx = assert_isa(shift, 'ksb::BuildContext'); my $fh = shift; my $deferredOptionsRef = shift; my @module_list; my $rcfile = $ctx->rcFile(); my ($option, %readModules); my $fileReader = ksb::RecursiveFH->new($rcfile); $fileReader->addFile($fh, $rcfile); # Read in global settings while ($_ = $fileReader->readLine()) { s/#.*$//; # Remove comments s/^\s+//; # Remove leading whitespace next unless $_; # Skip blank lines # First command in .kdesrc-buildrc should be a global # options declaration, even if none are defined. if (not /^global\s*$/) { error ("Invalid configuration file: $rcfile."); error ("Expecting global settings section at b[r[line $.]!"); die make_exception('Config', 'Missing global section'); } # Now read in each global option. _parseModuleOptions($ctx, $fileReader, $ctx); last; } my $using_default = 1; my %seenModules; # NOTE! *not* module-sets, *just* modules. my %seenModuleSets; # and vice versa -- named sets only though! my %seenModuleSetItems; # To track option override modules. # Now read in module settings while ($_ = $fileReader->readLine()) { s/#.*$//; # Remove comments s/^\s*//; # Remove leading whitespace next if (/^\s*$/); # Skip blank lines # Get modulename (has dash, dots, slashes, or letters/numbers) my ($type, $modulename) = /^(options|module)\s+([-\/\.\w]+)\s*$/; my $newModule; # 'include' directives can change the current file, so check where we're at $rcfile = $fileReader->currentFilename(); # Module-set? if (not $modulename) { my $moduleSetRE = qr/^module-set\s*([-\/\.\w]+)?\s*$/; ($modulename) = m/$moduleSetRE/; # modulename may be blank -- use the regex directly to match if (not /$moduleSetRE/) { error ("Invalid configuration file $rcfile!"); error ("Expecting a start of module section at r[b[line $.]."); die make_exception('Config', 'Ungrouped/Unknown option'); } if ($modulename && exists $seenModuleSets{$modulename}) { error ("Duplicate module-set $modulename at $rcfile:$."); die make_exception('Config', "Duplicate module-set $modulename defined at $rcfile:$."); } if ($modulename && exists $seenModules{$modulename}) { error ("Name $modulename for module-set at $rcfile:$. is already in use on a module"); die make_exception('Config', "Can't re-use name $modulename for module-set defined at $rcfile:$."); } # A moduleset can give us more than one module to add. $newModule = _parseModuleSetOptions($ctx, $fileReader, ksb::ModuleSet->new($ctx, $modulename || "")); # Save 'use-modules' entries so we can see if later module decls # are overriding/overlaying their options. my @moduleSetItems = $newModule->moduleNamesToFind(); @seenModuleSetItems{@moduleSetItems} = ($newModule) x scalar @moduleSetItems; $seenModuleSets{$modulename} = $newModule if $modulename; } # Duplicate module entry? (Note, this must be checked before the check # below for 'options' sets) elsif (exists $seenModules{$modulename} && $type ne 'options') { error ("Duplicate module declaration b[r[$modulename] on line $. of $rcfile"); die make_exception('Config', "Duplicate module $modulename declared at $rcfile:$."); } # Module/module-set options overrides elsif ($type eq 'options') { my $options = _parseModuleOptions($ctx, $fileReader, ksb::OptionsBase->new()); $deferredOptionsRef->{$modulename} = $options->{options}; next; # Don't add to module list } # Must follow 'options' handling elsif (exists $seenModuleSets{$modulename}) { error ("Name $modulename for module at $rcfile:$. is already in use on a module-set"); die make_exception('Config', "Can't re-use name $modulename for module defined at $rcfile:$."); } else { $newModule = _parseModuleOptions($ctx, $fileReader, ksb::Module->new($ctx, $modulename)); $seenModules{$modulename} = $newModule; } push @module_list, $newModule; $using_default = 0; } while (my ($name, $moduleSet) = each %seenModuleSets) { _validateModuleSet($ctx, $moduleSet); } # If the user doesn't ask to build any modules, build a default set. # The good question is what exactly should be built, but oh well. if ($using_default) { warning (" b[y[*] There do not seem to be any modules to build in your configuration."); return (); } return @module_list; } # Exits out of kdesrc-build, executing the user's preferred shell instead. The # difference is that the environment variables should be as set in kdesrc-build # instead of as read from .bashrc and friends. # # You should pass in the options to run the program with as a list. # # Meant to implement the --run command line option. sub _executeCommandLineProgram { my ($program, @args) = @_; if (!$program) { error ("You need to specify a program with the --run option."); exit 1; # Can't use finish here. } if (($< != $>) && ($> == 0)) { error ("kdesrc-build will not run a program as root unless you really are root."); exit 1; } debug ("Executing b[r[$program] ", join(' ', @args)); exit 0 if pretending(); exec $program, @args or do { # If we get to here, that sucks, but don't continue. error ("Error executing $program: $!"); exit 1; }; } # Function: _split_url # # Subroutine to split a url into a protocol and host sub _split_url { my $url = shift; my ($proto, $host) = ($url =~ m|([^:]*)://([^/]*)/|); return ($proto, $host); } # Function: _check_for_ssh_agent # # Checks if we are supposed to use ssh agent by examining the environment, and # if so checks if ssh-agent has a list of identities. If it doesn't, we run # ssh-add (with no arguments) and inform the user. This can be controlled with # the disable-agent-check parameter. # # Parameters: # 1. Build context sub _check_for_ssh_agent { my $ctx = assert_isa(shift, 'ksb::BuildContext'); # Don't bother with all this if the user isn't even using SSH. return 1 if pretending(); my @svnServers = grep { $_->scmType() eq 'svn' } ($ctx->modulesInPhase('update')); my @gitServers = grep { $_->scmType() eq 'git' } ($ctx->modulesInPhase('update')); my @sshServers = grep { my ($proto, $host) = _split_url($_->getOption('svn-server')); # Check if ssh is explicitly used in the proto, or if the host is the # developer main svn. (defined $proto && $proto =~ /ssh/) || (defined $host && $host =~ /^svn\.kde\.org/); } @svnServers; push @sshServers, grep { # Check for git+ssh:// or git@git.kde.org:/path/etc. my $repo = $_->getOption('repository'); ($repo =~ /^git\+ssh:\/\//) || ($repo =~ /^[a-zA-Z0-9_.]+@.*:\//); } @gitServers; return 1 if (not @sshServers) or $ctx->getOption('disable-agent-check'); whisper ("\tChecking for SSH Agent") if (scalar @sshServers); # We're using ssh to download, see if ssh-agent is running. return 1 unless exists $ENV{'SSH_AGENT_PID'}; my $pid = $ENV{'SSH_AGENT_PID'}; # It's supposed to be running, let's see if there exists the program with # that pid (this check is linux-specific at the moment). if (-d "/proc" and not -e "/proc/$pid") { warning ("r[ *] SSH Agent is enabled, but y[doesn't seem to be running]."); warning ("Since SSH is used to download from Subversion you may want to see why"); warning ("SSH Agent is not working, or correct the environment variable settings."); return 0; } # The agent is running, but does it have any keys? We can't be more specific # with this check because we don't know what key is required. my $noKeys = 0; filter_program_output(sub { $noKeys ||= /no identities/ }, 'ssh-add', '-l'); if ($noKeys) { # Use print so user can't inadvertently keep us quiet about this. print ksb::Debug::colorize (<getOption('ssh-identity-file'); push (@commandLine, $identFile) if $identFile; my $result = system (@commandLine); if ($result) # Run this code for both death-by-signal and nonzero return { my $rcfile = $ctx->rcFile(); print "\nUnable to add SSH identity, aborting.\n"; print "If you don't want kdesrc-build to check in the future,\n"; print ksb::Debug::colorize ("Set the g[disable-agent-check] option to g[true] in your $rcfile.\n\n"); return 0; } } return 1; } # Function: _handle_updates # # Subroutine to update a list of modules. Uses a Mojolicious event loop # to run each update in a subprocess to avoid blocking the script. Only # one update process will exist at a given time. # # Parameters: # 1. Build Context, which will be used to determine the module update list. # 2. A PromiseChain for adding work items and dependencies. # 3. A "start promise" that can be waited on for pre-update steps. # # This function accounts for every module in $ctx's update phase. # # Returns an updated start promise and can also throw exception on error sub _handle_updates { my ($ctx, $promiseChain, $start_promise) = @_; my $kdesrc = $ctx->getSourceDir(); my @update_list = $ctx->modulesInPhase('update'); return $start_promise unless @update_list; croak_runtime("SSH agent is not running but should be") unless _check_for_ssh_agent($ctx); # TODO: Extract this to a setup function that all updates/build depend upon whisper ("Creating source directory") unless -e $kdesrc; croak_runtime ("Unable to make directory r[$kdesrc]! $!") if (! -e $kdesrc && !super_mkdir ($kdesrc)); for my $module (@update_list) { # sub must be defined here to capture $module in the loop my $updateSub = sub { return $module->runPhase_p('update', # called in child process, can block sub { return $module->update($ctx) }, # called in this process, with results sub { my (undef, $was_successful, $extras) = @_; $module->setOption('#numUpdates', $extras->{update_count}); return $was_successful; } ); }; $promiseChain->addItem("$module/update", "network-queue", $updateSub); } return $start_promise; } # Throws an exception if essential build programs are missing as a sanity check. sub _checkForEarlyBuildExit { my $ctx = shift; my @modules = $ctx->modulesInPhase('build'); # Check for absolutely essential programs now. if (!_checkForEssentialBuildPrograms($ctx) && !exists $ENV{KDESRC_BUILD_IGNORE_MISSING_PROGRAMS}) { error (" r[b[*] Aborting now to save a lot of wasted time."); error (" y[b[*] export KDESRC_BUILD_IGNORE_MISSING_PROGRAMS=1 and re-run (perhaps with --no-src)"); error (" r[b[*] to continue anyways. If this check was in error please report a bug against"); error (" y[b[*] kdesrc-build at https://bugs.kde.org/"); croak_runtime ("Essential build programs are missing!"); } } sub _openStatusFileHandle { my $ctx = shift; my $outfile = pretending() ? '/dev/null' : $ctx->getLogDir() . '/build-status'; my $statusFile; open $statusFile, '>', $outfile or do { error (<modulesInPhase('build'); my $result = 0; _checkForEarlyBuildExit($ctx); # exception-thrower my $num_modules = scalar @modules; my ($statusFile, $outfile) = _openStatusFileHandle($ctx); my $everFailed = 0; # This generates a bunch of subs but doesn't call them yet foreach my $module (@modules) { # Needs to happen in this loop to capture $module my $buildSub = sub { return if ($everFailed && $module->getOption('stop-on-failure')); my $fail_count = $module->getPersistentOption('failure-count') // 0; my $num_updates = int ($module->getOption('#numUpdates', 'module') // 1); # check for skipped updates, --no-src forces build-when-unchanged # even when ordinarily disabled if ($num_updates == 0 && !$module->getOption('build-when-unchanged') && $fail_count == 0) { # TODO: Why is the param order reversed for these two? $ctx->statusMonitor()->markPhaseStart("$module", 'build'); $ctx->markModulePhaseSucceeded('build', $module); return 'skipped'; } # Can't build w/out blocking so return a promise instead, which ->build # already supplies return $module->build()->catch(sub { my $failureReason = shift; if (!$everFailed) { # No failures yet, mark this as resume point $everFailed = 1; my $moduleList = join(', ', map { "$_" } ($module, @modules)); $ctx->setPersistentOption('global', 'resume-list', $moduleList); } ++$fail_count; # Force this promise chain to stay dead return Mojo::Promise->new->reject('build'); })->then(sub { $fail_count = 0; })->finally(sub { $module->setPersistentOption('failure-count', $fail_count); }); }; $promiseChain->addItem("$module/build", 'cpu-queue', $buildSub); # If there's an update phase we need to depend on it and show status if (my $updatePromise = $promiseChain->promiseFor("$module/update")) { $promiseChain->addDep("$module/build", "$module/update"); } }; # Add to the build 'queue' for promise chain so that this runs only after all # other build jobs $promiseChain->addDep('@postBuild', 'cpu-queue', sub { if ($statusFile) { close $statusFile; # Update the symlink in latest to point to this file. my $logdir = $ctx->getSubdirPath('log-dir'); if (-l "$logdir/latest/build-status") { safe_unlink("$logdir/latest/build-status"); } symlink($outfile, "$logdir/latest/build-status"); } return Mojo::Promise->new->reject if $everFailed; return 0; }); return $start_promise->then( sub { $ctx->unsetPersistentOption('global', 'resume-list') }); } # Function: _handle_async_build # # This subroutine special-cases the handling of the update and build phases, by # performing them concurrently using forked processes and non-blocking I/O. # See Mojo::Promise and Mojo::IOLoop::Subprocess # # This procedure will use multiple processes (the main process and separate # processes for each update or build as they occur). # # Parameters: # 1. Build Context to use, from which the module lists will be determined. # # Returns 0 on success, non-zero on failure. sub _handle_async_build { my ($ctx) = @_; my $result = 0; $ctx->statusMonitor()->createBuildPlan($ctx); my $promiseChain = ksb::PromiseChain->new; my $start_promise = Mojo::Promise->new; # These succeed or die outright eval { $start_promise = _handle_updates ($ctx, $promiseChain, $start_promise); $start_promise = _handle_build ($ctx, $promiseChain, $start_promise); }; if ($@) { error ("Caught an error $@ setting up to build"); return 1; } my $chain = $promiseChain->makePromiseChain($start_promise) ->finally(sub { # Fail if we had a zero-valued result (indicates error) my @results = @_; # Must use ! here to make '0 but true' hack work $result = 1 if defined first { !($_->[0] // 1) } @results; $ctx->statusMonitor()->markBuildDone(); }); # Start the update/build process $start_promise->resolve; Mojo::IOLoop->stop; # Force the wait below to block $chain->wait; return $result; } # Function: _handle_install # # Handles the installation process. Simply calls 'make install' in the build # directory, though there is also provision for cleaning the build directory # afterwards, or stopping immediately if there is a build failure (normally # every built module is attempted to be installed). # # Parameters: # 1. Build Context, from which the install list is generated. # # Return value is a shell-style success code (0 == success) sub _handle_install { my $ctx = assert_isa(shift, 'ksb::BuildContext'); my @modules = $ctx->modulesInPhase('install'); @modules = grep { $_->buildSystem()->needsInstalled() } (@modules); my $result = 0; for my $module (@modules) { $ctx->resetEnvironment(); $result = $module->install() || $result; if ($result && $module->getOption('stop-on-failure')) { note ("y[Stopping here]."); return 1; # Error } } return $result; } # Function: _handle_uninstall # # Handles the uninstal process. Simply calls 'make uninstall' in the build # directory, while assuming that Qt or CMake actually handles it. # # The order of the modules is often significant, and it may work better to # uninstall modules in reverse order from how they were installed. However this # code does not automatically reverse the order; modules are uninstalled in the # order determined by the build context. # # This function obeys the 'stop-on-failure' option supported by _handle_install. # # Parameters: # 1. Build Context, from which the uninstall list is generated. # # Return value is a shell-style success code (0 == success) sub _handle_uninstall { my $ctx = assert_isa(shift, 'ksb::BuildContext'); my @modules = $ctx->modulesInPhase('uninstall'); @modules = grep { $_->buildSystem()->needsInstalled() } (@modules); my $result = 0; for my $module (@modules) { $ctx->resetEnvironment(); $result = $module->uninstall() || $result; if ($result && $module->getOption('stop-on-failure')) { note ("y[Stopping here]."); return 1; # Error } } return $result; } # Function: _applyModuleFilters # # Applies any module-specific filtering that is necessary after reading command # line and rc-file options. (This is as opposed to phase filters, which leave # each module as-is but change the phases they operate as part of, this # function could remove a module entirely from the build). # # Used for --resume-{from,after} and --stop-{before,after}, but more could be # added in theory. # This subroutine supports --{resume,stop}-* for both modules and module-sets. # # Parameters: # ctx - in use. # @modules - List of or to apply filters on. # # Returns: # list of or with any inclusion/exclusion filters # applied. Do not assume this list will be a strict subset of the input list, # however the order will not change amongst the input modules. sub _applyModuleFilters { my $ctx = assert_isa(shift, 'ksb::BuildContext'); my @moduleList = @_; if (!$ctx->getOption('resume-from') && !$ctx->getOption('resume-after') && !$ctx->getOption('stop-before') && !$ctx->getOption('stop-after')) { debug ("No command-line filter seems to be present."); return @moduleList; } if ($ctx->getOption('resume-from') && $ctx->getOption('resume-after')) { # This one's an error. error (<getOption('stop-before') && $ctx->getOption('stop-after')) { # This one's an error. error (<getOption('resume-from') || $ctx->getOption('resume-after'); my $startIndex = scalar @moduleList; if ($resumePoint) { debug ("Looking for $resumePoint for --resume-* option"); # || 0 is a hack to force Boolean context. my $filterInclusive = $ctx->getOption('resume-from') || 0; my $found = 0; for (my $i = 0; $i < scalar @moduleList; $i++) { my $module = $moduleList[$i]; $found = $module->name() eq $resumePoint; if ($found) { $startIndex = $filterInclusive ? $i : $i + 1; $startIndex = min($startIndex, scalar @moduleList - 1); last; } } } else { $startIndex = 0; } my $stopPoint = $ctx->getOption('stop-before') || $ctx->getOption('stop-after'); my $stopIndex = 0; if ($stopPoint) { debug ("Looking for $stopPoint for --stop-* option"); # || 0 is a hack to force Boolean context. my $filterInclusive = $ctx->getOption('stop-before') || 0; my $found = 0; for (my $i = $startIndex; $i < scalar @moduleList; $i++) { my $module = $moduleList[$i]; $found = $module->name() eq $stopPoint; if ($found) { $stopIndex = $i - ($filterInclusive ? 1 : 0); last; } } } else { $stopIndex = scalar @moduleList - 1; } if ($startIndex > $stopIndex || scalar @moduleList == 0) { # Lost all modules somehow. croak_runtime("Unknown resume -> stop point $resumePoint -> $stopPoint."); } return @moduleList[$startIndex .. $stopIndex]; } # Updates the built-in phase list for all Modules passed into this function in # accordance with the options set by the user. sub _updateModulePhases { whisper ("Filtering out module phases."); for my $module (@_) { if ($module->getOption('manual-update') || $module->getOption('no-svn') || $module->getOption('no-src')) { $module->phases()->clear(); next; } if ($module->getOption('manual-build')) { $module->phases()->filterOutPhase('build'); $module->phases()->filterOutPhase('test'); $module->phases()->filterOutPhase('install'); } $module->phases()->filterOutPhase('install') unless $module->getOption('install-after-build'); $module->phases()->addPhase('test') if $module->getOption('run-tests'); } return @_; } # This subroutine extract the value from options of the form --option=value, # which can also be expressed as --option value. # # The first parameter is the option that the user passed to the cmd line (e.g. # --prefix=/opt/foo). # The second parameter is a reference to the list of command line options. # # The return value is the value of the option (the list of options might be # shorter by 1, copy it if you don't want it to change), or undef if no value # was provided. sub _extractOptionValue { my ($option, $options_ref) = @_; if ($option =~ /=/) { my @value = split(/=/, $option); shift @value; # We don't need the first one, that the --option part. return if (scalar @value == 0); # If we have more than one element left in @value it's because the # option itself has an = in it, make sure it goes back in the answer. return join('=', @value); } return if scalar @{$options_ref} == 0; return shift @{$options_ref}; } # Like _extractOptionValue, but throws an exception if the value is not # actually present, so you don't have to check for it yourself. If you do get a # return value, it will be defined to something. sub _extractOptionValueRequired { my ($option, $options_ref) = @_; my $returnValue = _extractOptionValue($option, $options_ref); if (not defined $returnValue) { croak_runtime("Option $option needs to be set to some value instead of left blank"); } return $returnValue; } # Function: _cleanup_log_directory # # This function removes log directories from old kdesrc-build runs. All log # directories not referenced by $log_dir/latest somehow are made to go away. # # Parameters: # 1. Build context. # # No return value. sub _cleanup_log_directory { my $ctx = assert_isa(shift, 'ksb::BuildContext'); my $logdir = $ctx->getSubdirPath('log-dir'); return 0 if ! -e "$logdir/latest"; # Could happen for error on first run... # This glob relies on the date being in the specific format YYYY-MM-DD-ID my @dirs = bsd_glob("$logdir/????-??-??-??/", GLOB_NOSORT); my @needed = _reachableModuleLogs("$logdir/latest"); # Convert a list to a hash lookup since Perl lacks a "list-has" my %needed_table; @needed_table{@needed} = (1) x @needed; for my $dir (@dirs) { my ($id) = ($dir =~ m/(\d\d\d\d-\d\d-\d\d-\d\d)/); safe_rmtree($dir) unless $needed_table{$id}; } } # Function: _output_failed_module_list # # Print out an error message, and a list of modules that match that error # message. It will also display the log file name if one can be determined. # The message will be displayed all in uppercase, with PACKAGES prepended, so # all you have to do is give a descriptive message of what this list of # packages failed at doing. # # No message is printed out if the list of failed modules is empty, so this # function can be called unconditionally. # # Parameters: # 1. Build Context # 2. Message to print (e.g. 'failed to foo') # 3. List of ksb::Modules that had failed to foo # # No return value. sub _output_failed_module_list { my ($ctx, $message, @fail_list) = @_; assert_isa($ctx, 'ksb::BuildContext'); $message = uc $message; # Be annoying if (@fail_list) { debug ("Message is $message"); debug ("\tfor ", join(', ', @fail_list)); } if (scalar @fail_list > 0) { my $homedir = $ENV{'HOME'}; my $logfile; warning ("\nr[b[<<< PACKAGES $message >>>]"); for my $module (@fail_list) { $logfile = $module->getOption('#error-log-file'); # async updates may cause us not to have a error log file stored # (though this should now only happen due to other bugs). Since # there's only one place it should be, take advantage of # side-effect of log_command() to find it. if (not $logfile) { my $logdir = $module->getLogDir() . "/error.log"; $logfile = $logdir if -e $logdir; } $logfile = "No log file" unless $logfile; $logfile =~ s|$homedir|~|; warning ("r[$module]") if pretending(); warning ("r[$module] - g[$logfile]") if not pretending(); } } } # Function: _output_failed_module_lists # # This subroutine reads the list of failed modules for each phase in the build # context and calls _output_failed_module_list for all the module failures. # # Parameters: # 1. Build context # # Return value: # None sub _output_failed_module_lists { my $ctx = assert_isa(shift, 'ksb::BuildContext'); my $moduleGraph = shift; my $extraDebugInfo = { phases => {}, failCount => {} }; my @actualFailures = (); # This list should correspond to the possible phase names (although # it doesn't yet since the old code didn't, TODO) for my $phase ($ctx->phases()->phases()) { my @failures = $ctx->failedModulesInPhase($phase); for my $failure (@failures) { # we already tagged the failure before, should not happen but # make sure to check to avoid spurious duplicate output next if $extraDebugInfo->{phases}->{$failure}; $extraDebugInfo->{phases}->{$failure} = $phase; push @actualFailures, $failure; } _output_failed_module_list($ctx, "failed to $phase", @failures); } # See if any modules fail continuously and warn specifically for them. my @super_fail = grep { ($_->getPersistentOption('failure-count') // 0) > 3 } (@{$ctx->moduleList()}); if (@super_fail) { warning ("\nThe following modules have failed to build 3 or more times in a row:"); warning ("\tr[b[$_]") foreach @super_fail; warning ("\nThere is probably a local error causing this kind of consistent failure, it"); warning ("is recommended to verify no issues on the system.\n"); } my $top = 5; my $numSuggestedModules = scalar @actualFailures; # # Omit listing $top modules if there are that many or fewer anyway. # Not much point ranking 4 out of 4 failures, # this feature is meant for 5 out of 65 # if ($numSuggestedModules > $top) { my @sortedForDebug = ksb::DebugOrderHints::sortFailuresInDebugOrder( $moduleGraph, $extraDebugInfo, \@actualFailures ); info ("\nThe following top $top may be the most important to fix to " . "get the build to work, listed in order of 'probably most " . "interesting' to 'probably least interesting' failure:\n"); info ("\tr[b[$_]") foreach (@sortedForDebug[0..($top - 1)]); } } # Function: _installTemplatedFile # # This function takes a given file and a build context, and installs it to a # given location while expanding out template entries within the source file. # # The template language is *extremely* simple: <% foo %> is replaced entirely # with the result of $ctx->getOption(foo, 'no-inherit'). If the result # evaluates false for any reason than an exception is thrown. No quoting of # any sort is used in the result, and there is no way to prevent expansion of # something that resembles the template format. # # Multiple template entries on a line will be replaced. # # The destination file will be created if it does not exist. If the file # already exists then an exception will be thrown. # # Error handling: Any errors will result in an exception being thrown. # # Parameters: # 1. Pathname to the source file (use absolute paths) # 2. Pathname to the destination file (use absolute paths) # 3. Build context to use for looking up template values # # Return value: There is no return value. sub _installTemplatedFile { my ($sourcePath, $destinationPath, $ctx) = @_; assert_isa($ctx, 'ksb::BuildContext'); open (my $input, '<', $sourcePath) or croak_runtime("Unable to open template source $sourcePath: $!"); open (my $output, '>', $destinationPath) or croak_runtime("Unable to open template output $destinationPath: $!"); while (!eof ($input)) { my $line = readline($input); if (!defined ($line)) { croak_runtime("Failed to read from $sourcePath at line $.: $!"); unlink($destinationPath); } # Some lines should only be present in the source as they aid with testing. next if $line =~ /kdesrc-build: filter/; $line =~ s { <% \s* # Template bracket and whitespace ([^\s%]+) # Capture variable name \s*%> # remaining whitespace and closing bracket } { $ctx->getOption($1, 'module') // croak_runtime("Invalid variable $1") }gxe; # Replace all matching expressions, use extended regexp w/ # comments, and replacement is Perl code to execute. (print $output $line) or croak_runtime("Unable to write line to $destinationPath at line $.: $!"); } } # Function: _installCustomFile # # This function installs a source file to a destination path, assuming the # source file is a "templated" source file (see also _installTemplatedFile), and # records a digest of the file actually installed. This function will overwrite # a destination if the destination is identical to the last-installed file. # # Error handling: Any errors will result in an exception being thrown. # # Parameters: # 1. Build context to use for looking up template values, # 2. The full path to the source file. # 3. The full path to the destination file (incl. name) # 4. The key name to use for searching/recording installed MD5 digest. # # Return value: There is no return value. sub _installCustomFile { use File::Copy qw(copy); my $ctx = assert_isa(shift, 'ksb::BuildContext'); my ($sourceFilePath, $destFilePath, $md5KeyName) = @_; my $baseName = basename($sourceFilePath); if (-e $destFilePath) { my $existingMD5 = $ctx->getPersistentOption('/digests', $md5KeyName) // ''; if (fileDigestMD5($destFilePath) ne $existingMD5) { if (!$ctx->getOption('#delete-my-settings')) { error ("\tr[*] Installing \"b[$baseName]\" would overwrite an existing file:"); error ("\tr[*] y[b[$destFilePath]"); error ("\tr[*] If this is acceptable, please delete the existing file and re-run,"); error ("\tr[*] or pass b[--delete-my-settings] and re-run."); return; } elsif (!pretending()) { copy ($destFilePath, "$destFilePath.kdesrc-build-backup"); } } } if (!pretending()) { _installTemplatedFile($sourceFilePath, $destFilePath, $ctx); $ctx->setPersistentOption('/digests', $md5KeyName, fileDigestMD5($destFilePath)); } } # Function: _installCustomSessionDriver # # This function installs the included sample .xsession and environment variable # setup files, and records the md5sum of the installed results. # # If a file already exists, then its md5sum is taken and if the same as what # was previously installed, is overwritten. If not the same, the original file # is left in place and the .xsession is instead installed to # .xsession-kdesrc-build # # Error handling: Any errors will result in an exception being thrown. # # Parameters: # 1. Build context to use for looking up template values, # # Return value: There is no return value. sub _installCustomSessionDriver { use FindBin qw($RealBin); use List::Util qw(first); use File::Copy qw(copy); my $ctx = assert_isa(shift, 'ksb::BuildContext'); my @xdgDataDirs = split(':', $ENV{XDG_DATA_DIRS} || '/usr/local/share/:/usr/share/'); my $xdgDataHome = $ENV{XDG_DATA_HOME} || "$ENV{HOME}/.local/share"; # First we have to find the source my @searchPaths = ($RealBin, map { "$_/apps/kdesrc-build" } ($xdgDataHome, @xdgDataDirs)); s{/+$}{} foreach @searchPaths; # Remove trailing slashes s{//+}{/}g foreach @searchPaths; # Remove duplicate slashes my $envScript = first { -f $_ } ( map { "$_/sample-kde-env-master.sh" } @searchPaths ); my $sessionScript = first { -f $_ } ( map { "$_/sample-xsession.sh" } @searchPaths ); if (!$envScript || !$sessionScript) { warning ("b[*] Unable to find helper files to setup a login session."); warning ("b[*] You will have to setup login yourself, or install kdesrc-build properly."); return; } my $destDir = $ENV{XDG_CONFIG_HOME} || "$ENV{HOME}/.config"; super_mkdir($destDir) unless -d $destDir; _installCustomFile($ctx, $envScript, "$destDir/kde-env-master.sh", 'kde-env-master-digest'); _installCustomFile($ctx, $sessionScript, "$ENV{HOME}/.xsession", 'xsession-digest') if $ctx->getOption('install-session-driver'); if (!pretending()) { if ($ctx->getOption('install-session-driver') && !chmod (0744, "$ENV{HOME}/.xsession")) { error ("\tb[r[*] Error making b[~/.xsession] executable: $!"); error ("\tb[r[*] If this file is not executable you may not be able to login!"); }; } } # Function: _checkForEssentialBuildPrograms # # This subroutine checks for programs which are absolutely essential to the # *build* process and returns false if they are not all present. Right now this # just means qmake and cmake (although this depends on what modules are # actually present in the build context). # # Parameters: # 1. Build context # # Return value: # None sub _checkForEssentialBuildPrograms { my $ctx = assert_isa(shift, 'ksb::BuildContext'); my $kdedir = $ctx->getOption('kdedir'); my $qtdir = $ctx->getOption('qtdir'); my @preferred_paths = ("$kdedir/bin", "$qtdir/bin"); return 1 if pretending(); my @buildModules = $ctx->modulesInPhase('build'); my %requiredPrograms; my %modulesRequiringProgram; foreach my $module ($ctx->modulesInPhase('build')) { my @progs = $module->buildSystem()->requiredPrograms(); # Deliberately used @, since requiredPrograms can return a list. @requiredPrograms{@progs} = 1; foreach my $prog (@progs) { $modulesRequiringProgram{$prog} //= { }; $modulesRequiringProgram{$prog}->{$module->name()} = 1; } } my $wasError = 0; for my $prog (keys %requiredPrograms) { my %requiredPackages = ( qmake => 'Qt', cmake => 'CMake', meson => 'Meson', ); my $preferredPath = absPathToExecutable($prog, @preferred_paths); my $programPath = $preferredPath || absPathToExecutable($prog); # qmake is not necessarily named 'qmake' if (!$programPath && $prog eq 'qmake') { $programPath = ksb::BuildSystem::QMake::absPathToQMake(); } if (!$programPath) { # Don't complain about Qt if we're building it... if ($prog eq 'qmake' && ( grep { $_->buildSystemType() eq 'Qt' || $_->buildSystemType() eq 'Qt5' } (@buildModules)) || pretending() ) { next; } $wasError = 1; my $reqPackage = $requiredPackages{$prog} || $prog; my @modulesNeeding = keys %{$modulesRequiringProgram{$prog}}; local $, = ', '; # List separator in output error (<<"EOF"); Unable to find r[b[$prog]. This program is absolutely essential for building the modules: y[@modulesNeeding]. Please ensure the development packages for $reqPackage are installed by using your distribution's package manager. EOF } } return !$wasError; } # Function: _reachableModuleLogs # # Returns a list of module directory IDs that must be kept due to being # referenced from the "latest" symlink. # # This function may call itself recursively if needed. # # Parameters: # 1. The log directory under which to search for symlinks, including the "/latest" # part of the path. sub _reachableModuleLogs { my $logdir = shift; my @dirs; # A lexicalized var (my $foo) is required in face of recursiveness. opendir(my $fh, $logdir) or croak_runtime("Can't opendir $logdir: $!"); my $dir = readdir($fh); while(defined $dir) { if (-l "$logdir/$dir") { my $link = readlink("$logdir/$dir"); push @dirs, $link; } elsif ($dir !~ /^\.{1,2}$/) { # Skip . and .. directories (this is a great idea, trust me) push @dirs, _reachableModuleLogs("$logdir/$dir"); } $dir = readdir $fh; } closedir $fh; # Extract numeric IDs from directory names. @dirs = map { m/(\d{4}-\d\d-\d\d-\d\d)/ } (@dirs); # Convert to unique list by abusing hash keys. my %tempHash; @tempHash{@dirs} = (); return keys %tempHash; } # Installs the given subroutine as a signal handler for a set of signals which # could kill the program. # # First parameter is a reference to the sub to act as the handler. sub _installSignalHandlers { use Carp qw(confess); my $handlerRef = shift; my @signals = qw/HUP INT QUIT ABRT TERM PIPE/; foreach my $signal (@signals) { $SIG{$signal} = sub { confess ("Signal SIG$signal received, terminating.") unless $signal eq 'INT'; $handlerRef->(); }; } } # Ensures that basic one-time setup to actually *use* installed software is # performed, including .kdesrc-buildrc setup if necessary. # # Returns the appropriate exitcode to pass to the exit function sub performInitialUserSetup { my $self = shift; return ksb::FirstRun::setupUserSystem(); } # Shows a help message and version. Does not exit. sub _showHelpMessage { my $scriptVersion = scriptVersion(); say < and others, and is distributed under the terms of the GNU GPL v2. This script automates the download, build, and install process for KDE software using the latest available source code. Configuration is controlled from "\$PWD/kdesrc-buildrc" or "~/.kdesrc-buildrc". See kdesrc-buildrc-sample for an example. Usage: \$ $0 [--options] [module names] All configured modules are built if none are listed. Important Options: --pretend Don't actually take major actions, instead describe what would be done. --list-build List what modules would be built in the order in which they would be built. --dependency-tree Print out dependency information on the modules that would be built, using a `tree` format. Very useful for learning how modules relate to each other. May generate a lot of output. --no-src Don't update source code, just build/install. --src-only Only update the source code --refresh-build Start the build from scratch. --rc-file= Read configuration from filename instead of default. --initial-setup Installs Plasma env vars (~/.bashrc), required system pkgs, and a base kdesrc-buildrc. --resume-from= Skips modules until just before or after the given --resume-after= package, then operates as normal. --stop-before= Stops just before or after the given package is --stop-after= reached. --include-dependencies Also builds KDE-based dependencies of given modules. (This is enabled by default; use --no-include-dependencies to disable) --stop-on-failure Stops the build as soon as a package fails to build. More docs at https://docs.kde.org/trunk5/en/extragear-utils/kdesrc-build/ Supported configuration options: https://go.kde.org/u/ksboptions Supported cmdline options: https://go.kde.org/u/ksbcmdline DONE # Look for indications this is the first run. if (! -e "./kdesrc-buildrc" && ! -e "$ENV{HOME}/.kdesrc-buildrc") { say <{context}; } sub metadataModule { my $self = shift; return $self->{metadata_module}; } sub runMode { my $self = shift; return $self->{run_mode}; } sub modules { my $self = shift; return @{$self->{modules}}; } sub workLoad { my $self = shift; return $self->{workLoad}; } 1; diff --git a/modules/ksb/DependencyResolver.pm b/modules/ksb/DependencyResolver.pm index 86d29dd..28841d7 100644 --- a/modules/ksb/DependencyResolver.pm +++ b/modules/ksb/DependencyResolver.pm @@ -1,826 +1,768 @@ package ksb::DependencyResolver; # Class: DependencyResolver # # This module handles resolving dependencies between modules. Each "module" # from the perspective of this resolver is simply a module full name, as # given by the KDE Project database. (e.g. extragear/utils/kdesrc-build) use strict; use warnings; use 5.014; our $VERSION = '0.20'; use ksb::BuildException; use ksb::Debug; use ksb::Util; use List::Util qw(first); sub uniq { my %seen; return grep { ++($seen{$_}) == 1 } @_; } # Constructor: new # # Constructs a new . # # Parameters: # # moduleFactoryRef - Reference to a sub that creates ksb::Modules from # kde-project module names. Used for ksb::Modules for which the user # requested recursive dependency inclusion. # # Synposis: # # > my $resolver = new DependencyResolver($modNewRef); # > $resolver->readDependencyData(open my $fh, '<', 'file.txt'); # > $resolver->resolveDependencies(@modules); sub new { my $class = shift; my $moduleFactoryRef = shift; my $self = { # hash table mapping short module names (m) to a hashref key by branch # name, the value of which is yet another hashref (see # readDependencyData). Note that this assumes KDE git infrastructure # ensures that all full module names (e.g. # kde/workspace/plasma-workspace) map to a *unique* short name (e.g. # plasma-workspace) by stripping leading path components dependenciesOf => { }, # hash table mapping a wildcarded module name with no branch to a # listref of module:branch dependencies. catchAllDependencies => { }, # reference to a sub that will properly create a ksb::Module from a # given kde-project module name. Used to support automatically adding # dependencies to a build. moduleFactoryRef => $moduleFactoryRef, }; return bless $self, $class; } # Function: shortenModuleName # # Internal: # # This method returns the 'short' module name of kde-project full project paths. # E.g. 'kde/kdelibs/foo' would be shortened to 'foo'. # # This is a static function, not an object method. # # Parameters: # # path - A string holding the full module virtual path # # Returns: # # The module name. sub _shortenModuleName { my $name = shift; $name =~ s{^.*/}{}; # Uses greedy capture by default return $name; } # Method: readDependencyData # # Reads in dependency data in a pseudo-Makefile format. # See kde-build-metadata/dependency-data. # # Parameters: # $self - The DependencyResolver object. # $fh - Filehandle to read dependencies from (should already be open). # # Exceptions: # Can throw an exception on I/O errors or malformed dependencies. sub readDependencyData { my $self = assert_isa(shift, 'ksb::DependencyResolver'); my $fh = shift; my $dependenciesOfRef = $self->{dependenciesOf}; my $dependencyAtom = qr/ ^\s* # Clear leading whitespace ([^\[:\s]+) # (1) Capture anything not a [, :, or whitespace (dependent item) \s* # Clear whitespace we didn't capture (?:\[ # Open a non-capture group... ([^\]:\s]+) # (2) Capture branch name without brackets ])?+ # Close group, make optional, no backtracking \s* # Clear whitespace we didn't capture : \s* ([^\s\[]+) # (3) Capture all non-whitespace (source item) (?:\s*\[ # Open a non-capture group... ([^\]\s]+) # (4) Capture branch name without brackets ])?+ # Close group, make optional, no backtracking \s*$ # Ensure no trailing cruft. Any whitespace should end line /x; # /x Enables extended whitespace mode while(my $line = <$fh>) { # Strip comments, skip empty lines. $line =~ s{#.*$}{}; next if $line =~ /^\s*$/; if ($line !~ $dependencyAtom) { croak_internal("Invalid line $line when reading dependency data."); } my ($dependentItem, $dependentBranch, $sourceItem, $sourceBranch) = $line =~ $dependencyAtom; # Ignore "catch-all" dependencies where the source is the catch-all if ($sourceItem =~ m,\*$,) { warning ("\tIgnoring dependency on wildcard module grouping " . "on line $. of kde-build-metadata/dependency-data"); next; } $dependentBranch ||= '*'; # If no branch, apply catch-all flag $sourceBranch ||= '*'; # _shortenModuleName may remove negation marker so check now my $depKey = (index($sourceItem, '-') == 0) ? '-' : '+'; $sourceItem =~ s/^-//; # remove negation marker if name already short # Source can never be a catch-all so we can shorten early. Also, # we *must* shorten early to avoid a dependency on a long path. $sourceItem = _shortenModuleName($sourceItem); # Handle catch-all dependent groupings if ($dependentItem =~ /\*$/) { $self->{catchAllDependencies}->{$dependentItem} //= [ ]; push @{$self->{catchAllDependencies}->{$dependentItem}}, "$sourceItem:$sourceBranch"; next; } $dependentItem = _shortenModuleName($dependentItem); # Initialize with hashref if not already defined. The hashref will hold # - => [ ] (list of explicit *NON* dependencies of item:$branch), # + => [ ] (list of dependencies of item:$branch) # # Each dependency item is tracked at the module:branch level, and there # is always at least an entry for module:*, where '*' means branch # is unspecified and should only be used to add dependencies, never # take them away. # # Finally, all (non-)dependencies in a list are also of the form # fullname:branch, where "*" is a valid branch. $dependenciesOfRef->{"$dependentItem:*"} //= { '-' => [ ], '+' => [ ], }; # Create actual branch entry if not present $dependenciesOfRef->{"$dependentItem:$dependentBranch"} //= { '-' => [ ], '+' => [ ], }; push @{$dependenciesOfRef->{"$dependentItem:$dependentBranch"}->{$depKey}}, "$sourceItem:$sourceBranch"; } $self->_canonicalizeDependencies(); } # Function: _canonicalizeDependencies # # Ensures that all stored dependencies are stored in a way that allows for # reproducable dependency ordering (assuming the same dependency items and same # selectors are used). # # Parameters: none # # Returns: none sub _canonicalizeDependencies { my $self = shift; my $dependenciesOfRef = $self->{dependenciesOf}; foreach my $dependenciesRef (values %{$dependenciesOfRef}) { @{$dependenciesRef->{'-'}} = sort @{$dependenciesRef->{'-'}}; @{$dependenciesRef->{'+'}} = sort @{$dependenciesRef->{'+'}}; } } sub _lookupDirectDependencies { my $self = assert_isa(shift, 'ksb::DependencyResolver'); my ($path, $branch) = @_; my $dependenciesOfRef = $self->{dependenciesOf}; my @directDeps = (); my @exclusions = (); my $item = _shortenModuleName($path); my $moduleDepEntryRef = $dependenciesOfRef->{"$item:*"}; if ($moduleDepEntryRef) { debug("handling dependencies for: $item without branch (*)"); push @directDeps, @{$moduleDepEntryRef->{'+'}}; push @exclusions, @{$moduleDepEntryRef->{'-'}}; } if ($branch && $branch ne '*') { $moduleDepEntryRef = $dependenciesOfRef->{"$item:$branch"}; if ($moduleDepEntryRef) { debug("handling dependencies for: $item with branch ($branch)"); push @directDeps, @{$moduleDepEntryRef->{'+'}}; push @exclusions, @{$moduleDepEntryRef->{'-'}}; } } while (my ($catchAll, $deps) = each %{$self->{catchAllDependencies}}) { my $prefix = $catchAll; $prefix =~ s/\*$//; if (($path =~ /^$prefix/) || !$prefix) { push @directDeps, @{$deps}; } } foreach my $exclusion (@exclusions) { # Remove only modules at the exact given branch as a dep. # However catch-alls can remove catch-alls. # But catch-alls cannot remove a specific branch, such exclusions have # to also be specific. @directDeps = grep { $_ ne $exclusion } (@directDeps); } my $result = { syntaxErrors => 0, trivialCycles => 0, dependencies => {} }; for my $dep (@directDeps) { my ($depPath, $depBranch) = ($dep =~ m/^([^:]+):(.*)$/); if (!$depPath) { error("r[Invalid dependency declaration: b[$dep]]"); ++($result->{syntaxErrors}); next; } my $depItem = _shortenModuleName($depPath); if ($depItem eq $item) { debug("\tBreaking trivial cycle of b[$depItem] -> b[$item]"); ++($result->{trivialCycles}); next; } if ($result->{dependencies}->{$depItem}) { debug("\tSkipping duplicate direct dependency b[$depItem] of b[$item]"); } else { $depBranch //= ''; # work-around: wildcard branches are a don't care, not an actual # branch name/value $depBranch = undef if ($depBranch eq '' || $depBranch eq '*'); $result->{dependencies}->{$depItem} = { item => $depItem, path => $depPath, branch => $depBranch }; } } return $result; } sub _runDependencyVote { my $moduleGraph = shift; for my $item (keys(%$moduleGraph)) { my @names = keys(%{$moduleGraph->{$item}->{allDeps}->{items}}); for my $name (@names) { ++($moduleGraph->{$name}->{votes}->{$item}); } } return $moduleGraph; } sub _detectDependencyCycle { my ($moduleGraph, $depItem, $item) = @_; my $depModuleGraph = $moduleGraph->{$depItem}; if ($depModuleGraph->{traces}->{status}) { if ($depModuleGraph->{traces}->{status} == 2) { debug("Already resolved $depItem -- skipping"); return $depModuleGraph->{traces}->{result}; } else { error("Found a dependency cycle at: $depItem while tracing $item"); $depModuleGraph->{traces}->{result} = 1; } } else { $depModuleGraph->{traces}->{status} = 1; $depModuleGraph->{traces}->{result} = 0; my @names = keys(%{$depModuleGraph->{deps}}); for my $name (@names) { if (_detectDependencyCycle($moduleGraph, $name, $item)) { $depModuleGraph->{traces}->{result} = 1; } } } $depModuleGraph->{traces}->{status} = 2; return $depModuleGraph->{traces}->{result}; } sub _checkDependencyCycles { my $moduleGraph = shift; my $errors = 0; for my $item (keys(%$moduleGraph)) { if(_detectDependencyCycle($moduleGraph, $item, $item)) { error("Somehow there is a circular dependency involving b[$item]! :("); error("Please file a bug against kde-build-metadata about this!"); ++$errors; } } return $errors; } sub _copyUpDependenciesForModule { my ($moduleGraph, $item) = @_; my $allDeps = $moduleGraph->{$item}->{allDeps}; if($allDeps->{done}) { debug("\tAlready copied up dependencies for b[$item] -- skipping"); } else { debug("\tCopying up dependencies and transitive dependencies for item: b[$item]"); $allDeps->{items} = {}; my @names = keys(%{$moduleGraph->{$item}->{deps}}); for my $name (@names) { if ($allDeps->{items}->{$name}) { debug("\tAlready copied up (transitive) dependency on b[$name] for b[$item] -- skipping"); } else { _copyUpDependenciesForModule($moduleGraph, $name); my @copied = keys(%{$moduleGraph->{$name}->{allDeps}->{items}}); for my $copy (@copied) { if ($allDeps->{items}->{$copy}) { debug("\tAlready copied up (transitive) dependency on b[$copy] for b[$item] -- skipping"); } else { ++($allDeps->{items}->{$copy}); } } ++($allDeps->{items}->{$name}); } } ++($allDeps->{done}); } } sub _copyUpDependencies { my $moduleGraph = shift; for my $item (keys(%$moduleGraph)) { _copyUpDependenciesForModule($moduleGraph, $item); } return $moduleGraph; } sub _detectBranchConflict { my ($moduleGraph, $item, $branch) = @_; if ($branch) { my $subGraph = $moduleGraph->{$item}; my $previouslySelectedBranch = $subGraph->{branch}; return $previouslySelectedBranch if($previouslySelectedBranch && $previouslySelectedBranch ne $branch); } return undef; } sub _getDependencyPathOf { my ($module, $item, $path) = @_; if ($module) { my $projectPath = $module->fullProjectPath(); $projectPath = "third-party/$projectPath" if(!$module->isKDEProject()); debug("\tUsing path: 'b[$projectPath]' for item: b[$item]"); return $projectPath; } debug("\tGuessing path: 'b[$path]' for item: b[$item]"); return $path; } sub _resolveDependenciesForModuleDescription { my $self = assert_isa(shift, 'ksb::DependencyResolver'); my ($moduleGraph, $moduleDesc) = @_; my $module = $moduleDesc->{module}; if($module) { assert_isa($module, 'ksb::Module'); } my $path = $moduleDesc->{path}; my $item = $moduleDesc->{item}; my $branch = $moduleDesc->{branch}; my $prettyBranch = $branch ? "$branch" : "*"; my $includeDependencies = $module ? $module->getOption('include-dependencies') : $moduleDesc->{includeDependencies}; my $errors = { syntaxErrors => 0, trivialCycles => 0, branchErrors => 0 }; debug("Resolving dependencies for module: b[$item]"); while (my ($depItem, $depInfo) = each %{$moduleGraph->{$item}->{deps}}) { my $depPath = $depInfo->{path}; my $depBranch = $depInfo->{branch}; my $prettyDepBranch = $depBranch ? "$depBranch" : "*"; debug ("\tdep-resolv: b[$item:$prettyBranch] depends on b[$depItem:$prettyDepBranch]"); my $depModuleGraph = $moduleGraph->{$depItem}; if($depModuleGraph) { my $previouslySelectedBranch = _detectBranchConflict($moduleGraph, $depItem, $depBranch); if($previouslySelectedBranch) { error("r[Found a dependency conflict in branches ('b[$previouslySelectedBranch]' is not 'b[$prettyDepBranch]') for b[$depItem]! :("); ++($errors->{branchErrors}); } else { if($depBranch) { $depModuleGraph->{branch} = $depBranch; } } } else { my $depModule = $self->{moduleFactoryRef}($depItem); my $resolvedPath = _getDependencyPathOf($depModule, $depItem, $depPath); # May not exist, e.g. misspellings or 'virtual' dependencies like kf5umbrella. if(!$depModule) { debug("\tdep-resolve: Will not build virtual or undefined module: b[$depItem]\n"); } my $depLookupResult = $self->_lookupDirectDependencies( $resolvedPath, $depBranch ); $errors->{trivialCycles} += $depLookupResult->{trivialCycles}; $errors->{syntaxErrors} += $depLookupResult->{syntaxErrors}; $moduleGraph->{$depItem} = { votes => {}, path => $resolvedPath, build => $depModule && $includeDependencies ? 1 : 0, branch => $depBranch, deps => $depLookupResult->{dependencies}, allDeps => {}, module => $depModule, traces => {} }; my $depModuleDesc = { includeDependencies => $includeDependencies, module => $depModule, item => $depItem, path => $resolvedPath, branch => $depBranch }; if (!$moduleGraph->{$depItem}->{build}) { debug (" y[b[*] $item depends on $depItem, but no module builds $depItem for this run.]"); } if($depModule && $depBranch && (_getBranchOf($depModule) // '') ne "$depBranch") { my $wrongBranch = _getBranchOf($depModule) // '?'; error(" r[b[*] $item needs $depItem:$prettyDepBranch, not $depItem:$wrongBranch]"); ++($errors->{branchErrors}); } debug("Resolving transitive dependencies for module: b[$item] (via: b[$depItem:$prettyDepBranch])"); my $resolvErrors = $self->_resolveDependenciesForModuleDescription( $moduleGraph, $depModuleDesc ); $errors->{branchErrors} += $resolvErrors->{branchErrors}; $errors->{syntaxErrors} += $resolvErrors->{syntaxErrors}; $errors->{trivialCycles} += $resolvErrors->{trivialCycles}; } } return $errors; } sub resolveToModuleGraph { my $self = assert_isa(shift, 'ksb::DependencyResolver'); my @modules = @_; my %graph; my $moduleGraph = \%graph; my $result = { graph => $moduleGraph, errors => { branchErrors => 0, pathErrors => 0, trivialCycles => 0, syntaxErrors => 0, cycles => 0 } }; my $errors = $result->{errors}; for my $module (@modules) { my $item = $module->name(); # _shortenModuleName($path); my $branch = _getBranchOf($module); my $path = _getDependencyPathOf($module, $item, ''); if (!$path) { error("r[Unable to determine project/dependency path of module: $item]"); ++($errors->{pathErrors}); next; } if($moduleGraph->{$item}) { debug("Module pulled in previously through (transitive) dependencies: $item"); my $previouslySelectedBranch = _detectBranchConflict($moduleGraph, $item, $branch); if($previouslySelectedBranch) { error("r[Found a dependency conflict in branches ('b[$previouslySelectedBranch]' is not 'b[$branch]') for b[$item]! :("); ++($errors->{branchErrors}); } elsif ($branch) { $moduleGraph->{$item}->{branch} = $branch; } # # May have been pulled in via dependencies but not yet marked for # build. Do so now, since it is listed explicitly in @modules # $moduleGraph->{$item}->{build} = 1; } else { my $depLookupResult = $self->_lookupDirectDependencies( $path, $branch ); $errors->{trivialCycles} += $depLookupResult->{trivialCycles}; $errors->{syntaxErrors} += $depLookupResult->{syntaxErrors}; $moduleGraph->{$item} = { votes => {}, path => $path, build => 1, branch => $branch, module => $module, deps => $depLookupResult->{dependencies}, allDeps => {}, traces => {} }; my $moduleDesc = { includeDependencies => $module->getOption('include-dependencies'), path => $path, item => $item, branch => $branch, module => $module }; my $resolvErrors = $self->_resolveDependenciesForModuleDescription( $moduleGraph, $moduleDesc ); $errors->{branchErrors} += $resolvErrors->{branchErrors}; $errors->{syntaxErrors} += $resolvErrors->{syntaxErrors}; $errors->{trivialCycles} += $resolvErrors->{trivialCycles}; } } my $pathErrors = $errors->{pathErrors}; if ($pathErrors) { error("Total of items which were not resolved due to path lookup failure: $pathErrors"); } my $branchErrors = $errors->{branchErrors}; if ($branchErrors) { error("Total of branch conflicts detected: $branchErrors"); } my $syntaxErrors = $errors->{syntaxErrors}; if ($syntaxErrors) { error("Total of encountered syntax errors: $syntaxErrors"); } if ($syntaxErrors || $pathErrors || $branchErrors) { error("Unable to resolve dependency graph"); $result->{graph} = undef; return $result; } my $trivialCycles = $errors->{trivialCycles}; if ($trivialCycles) { warning("Total of 'trivial' dependency cycles detected & eliminated: $trivialCycles"); } my $cycles = _checkDependencyCycles($moduleGraph); if ($cycles) { error("Total of items with at least one circular dependency detected: $errors"); error("Unable to resolve dependency graph"); $result->{cycles} = $cycles; $result->{graph} = undef; return $result; } else { $result->{graph} = _runDependencyVote(_copyUpDependencies($moduleGraph)); return $result; } } -sub _descendModuleGraph +sub hasErrors { - my ($moduleGraph, $callback, $nodeInfo, $context) = @_; - - my $depth = $nodeInfo->{depth}; - my $index = $nodeInfo->{idx}; - my $count = $nodeInfo->{count}; - my $currentItem = $nodeInfo->{currentItem}; - my $currentBranch = $nodeInfo->{currentBranch}; - my $parentItem = $nodeInfo->{parentItem}; - my $parentBranch = $nodeInfo->{parentBranch}; - - my $subGraph = $moduleGraph->{$currentItem}; - &$callback($nodeInfo, $subGraph->{module}, $context); - - ++$depth; - - my @items = keys(%{$subGraph->{deps}}); - - my $itemCount = scalar(@items); - my $itemIndex = 1; - - for my $item (@items) - { - $subGraph = $moduleGraph->{$item}; - my $branch = $subGraph->{branch} // ''; - my $itemInfo = { - build => $subGraph->{build}, - depth => $depth, - idx => $itemIndex, - count => $itemCount, - currentItem => $item, - currentBranch => $branch, - parentItem => $currentItem, - parentBranch => $currentBranch - }; - _descendModuleGraph($moduleGraph, $callback, $itemInfo, $context); - ++$itemIndex; - } -} + my $info = shift; -sub walkModuleDependencyTrees -{ - my $moduleGraph = shift; - my $callback = shift; - my $context = shift; - my @modules = @_; - my $itemCount = scalar(@modules); - my $itemIndex = 1; + my $cycles = $info->{cycles} // 0; + my $pathErrors = $info->{pathErrors} // 0; + my $branchErrors = $info->{branchErrors} // 0; + my $syntaxErrors = $info->{syntaxErrors} // 0; - for my $module (@modules) { - assert_isa($module, 'ksb::Module'); - my $item = $module->name(); - my $subGraph = $moduleGraph->{$item}; - my $branch = $subGraph->{branch} // ''; - my $info = { - build => $subGraph->{build}, - depth => 0, - idx => $itemIndex, - count => $itemCount, - currentItem => $item, - currentBranch => $branch, - parentItem => '', - parentBranch => '' - }; - _descendModuleGraph($moduleGraph, $callback, $info, $context); - ++$itemIndex; - } + return $cycles || $pathErrors || $branchErrors || $syntaxErrors; } sub _compareBuildOrder { my ($moduleGraph, $a, $b) = @_; # # Enforce a strict dependency ordering. # The case where both are true should never happen, since that would # amount to a cycle, and cycle detection is supposed to have been # performed beforehand. # my $bDependsOnA = $moduleGraph->{$a}->{votes}->{$b} // 0; my $aDependsOnB = $moduleGraph->{$b}->{votes}->{$a} // 0; my $order = $bDependsOnA ? -1 : ($aDependsOnB ? 1 : 0); return $order if $order; # # Assuming no dependency relation, next sort by 'popularity': # the item with the most votes (back edges) is depended on the most # so it is probably a good idea to build that one earlier to help # maximise the duration of time for which builds can be run in parallel # my $voteA = scalar keys %{$moduleGraph->{$a}->{votes}}; my $voteB = scalar keys %{$moduleGraph->{$b}->{votes}}; my $votes = $voteB <=> $voteA; return $votes if $votes; # # If there is no good reason to perfer one module over another, # simply sort by name to get a reproducible build order. # That simplifies autotesting and/or reproducible builds. # (The items to sort are supplied as a hash so the order of keys is by # definition not guaranteed.) # my $name = ($a cmp $b); return $name; } sub sortModulesIntoBuildOrder { my $moduleGraph = shift; my @resolved = keys(%{$moduleGraph}); my @built = grep { $moduleGraph->{$_}->{build} && $moduleGraph->{$_}->{module} } (@resolved); my @prioritised = sort { _compareBuildOrder($moduleGraph, $a, $b); } (@built); my @modules = map { $moduleGraph->{$_}->{module} } (@prioritised); return @modules; } # Function: getBranchOf # # Internal: # # This function extracts the branch of the given Module by calling its # scm object's branch-determining method. It also ensures that the branch # returned was really intended to be a branch (as opposed to a detached HEAD); # undef is returned when the desired commit is not a branch name, otherwise # the user-requested branch name is returned. sub _getBranchOf { my $module = shift; my $scm = $module->scm(); # when the module's SCM is not git, # assume the default "no particular" branch wildcard return undef unless $scm->isa('ksb::Updater::Git'); my ($branch, $type) = $scm->_determinePreferredCheckoutSource($module); return ($type eq 'branch' ? $branch : undef); } 1; diff --git a/modules/ksb/UserInterface/DependencyGraph.pm b/modules/ksb/UserInterface/DependencyGraph.pm new file mode 100644 index 0000000..9270630 --- /dev/null +++ b/modules/ksb/UserInterface/DependencyGraph.pm @@ -0,0 +1,153 @@ +package ksb::UserInterface::DependencyGraph; + +use strict; +use warnings; +use 5.014; + +sub _descendModuleGraph +{ + my ($moduleGraph, $callback, $nodeInfo, $context) = @_; + + my $depth = $nodeInfo->{depth}; + my $index = $nodeInfo->{idx}; + my $count = $nodeInfo->{count}; + my $currentItem = $nodeInfo->{currentItem}; + my $currentBranch = $nodeInfo->{currentBranch}; + my $parentItem = $nodeInfo->{parentItem}; + my $parentBranch = $nodeInfo->{parentBranch}; + + my $subGraph = $moduleGraph->{$currentItem}; + &$callback($nodeInfo, $subGraph->{module}, $context); + + ++$depth; + + my @items = @{$subGraph->{deps}}; + + my $itemCount = scalar(@items); + my $itemIndex = 1; + + for my $item (@items) + { + $subGraph = $moduleGraph->{$item}; + my $branch = $subGraph->{branch} // ''; + my $itemInfo = { + build => $subGraph->{build}, + depth => $depth, + idx => $itemIndex, + count => $itemCount, + currentItem => $item, + currentBranch => $branch, + parentItem => $currentItem, + parentBranch => $currentBranch + }; + _descendModuleGraph($moduleGraph, $callback, $itemInfo, $context); + ++$itemIndex; + } +} + +sub _walkModuleDependencyTrees +{ + my $moduleGraph = shift; + my $callback = shift; + my $context = shift; + my @modules = @_; + my $itemCount = scalar(@modules); + my $itemIndex = 1; + + for my $item (@modules) { + my $subGraph = $moduleGraph->{$item}; + my $branch = $subGraph->{branch} // ''; + my $info = { + build => $subGraph->{build}, + depth => 0, + idx => $itemIndex, + count => $itemCount, + currentItem => $item, + currentBranch => $branch, + parentItem => '', + parentBranch => '' + }; + _descendModuleGraph($moduleGraph, $callback, $info, $context); + ++$itemIndex; + } +} + + +sub _treeOutputConnectors +{ + my ($depth, $index, $count) = @_; + my $blankPadding = (' ' x 4); + + return (' ── ', $blankPadding) if ($depth == 0); + return ('└── ', $blankPadding) if ($index == $count); + return ('├── ', '│ '); +} + +sub _yieldModuleDependencyTreeEntry +{ + my ($nodeInfo, $module, $context) = @_; + + my $depth = $nodeInfo->{depth}; + my $index = $nodeInfo->{idx}; + my $count = $nodeInfo->{count}; + my $build = $nodeInfo->{build}; + my $currentItem = $nodeInfo->{currentItem}; + my $currentBranch = $nodeInfo->{currentBranch}; + my $parentItem = $nodeInfo->{parentItem}; + my $parentBranch = $nodeInfo->{parentBranch}; + + my $buildStatus = $build ? 'built' : 'not built'; + my $statusInfo = $currentBranch ? "($buildStatus: $currentBranch)" : "($buildStatus)"; + + my $connectorStack = $context->{stack}; + my $prefix = pop(@$connectorStack); + + while($context->{depth} > $depth) { + $prefix = pop(@$connectorStack); + --($context->{depth}); + } + + push(@$connectorStack, $prefix); + + my ($connector, $padding) = _treeOutputConnectors($depth, $index, $count); + + push(@$connectorStack, $prefix . $padding); + $context->{depth} = $depth + 1; + + my $line = $prefix . $connector . $currentItem . ' ' . $statusInfo; + $context->{report}($line); +} + +sub printTrees +{ + my $tree = shift; + my @modules = @_; + + # + # Hack: reopen STDOUT to get rid of ... "does not map to ascii" noise + # Yes: the code points do not map to ASCII, that is sort of the point + # + my $ok = open my $fh, '>&', STDOUT; + return 1 unless $ok; + + my $depTreeCtx = { + stack => [''], + depth => 0, + report => sub { + my $line = shift; + print $fh $line, "\n"; + } + }; + + _walkModuleDependencyTrees( + $tree, + \&_yieldModuleDependencyTreeEntry, + $depTreeCtx, + @modules + ); + + close $fh; + return 0; +} + +1; diff --git a/modules/ksb/UserInterface/TTY.pm b/modules/ksb/UserInterface/TTY.pm index edd1cc8..ae3bcc0 100755 --- a/modules/ksb/UserInterface/TTY.pm +++ b/modules/ksb/UserInterface/TTY.pm @@ -1,284 +1,339 @@ #!/usr/bin/env perl package ksb::UserInterface::TTY 0.10; =pod =head1 NAME ksb::UserInterface::TTY -- A command-line interface to the kdesrc-build backend =head1 DESCRIPTION This class is used to show a user interface for a kdesrc-build run at the command line (as opposed to a browser-based or GUI interface). Since the kdesrc-build backend is now meant to be headless and controlled via a Web-style API set (powered by Mojolicious), this class manages the interaction with that backend, also using Mojolicious to power the HTTP and WebSocket requests necessary. =head1 SYNOPSIS my $app = web::BackendServer->new(@ARGV); my $ui = ksb::UserInterface::TTY->new($app); exit $ui->start(); # Blocks! Returns a shell-style return code =cut use strict; use warnings; use 5.014; use Mojo::Base -base; use Mojo::Server::Daemon; use Mojo::IOLoop; use Mojo::UserAgent; use Mojo::JSON qw(to_json); use ksb::BuildException; use ksb::StatusView; use ksb::Util; use ksb::Debug; +use ksb::UserInterface::DependencyGraph; +use Mojo::Promise; use IO::Handle; # For methods on event_stream file use List::Util qw(max); has ua => sub { Mojo::UserAgent->new->inactivity_timeout(0) }; has ui => sub { ksb::StatusView->new() }; has 'app'; sub new { my ($class, $app) = @_; my $self = $class->SUPER::new(app => $app); # Mojo::UserAgent can be tied to a Mojolicious application server directly to # handle relative URLs, which is perfect for what we want. Making this # attachment will startup the Web server behind the scenes and allow $ua to # make HTTP requests. $self->ua->server->app($app); # $self->ua->server->app->log->level('debug'); $self->ua->server->app->log->level('fatal'); return $self; } sub _check_error { my $tx = shift; my $err = $tx->error or return $tx; my $body = $tx->res->body // ''; open my $fh, '<', \$body; my ($first_line) = <$fh> // ''; $err->{message} .= "\n$first_line" if $first_line; die $err; }; + +sub _fetchModuleList +{ + my ($ua, $list) = @_; + return $ua->get_p($list)->then(sub { + my $tx = _check_error(shift); + return $tx->result->json; + }); +} + +sub dumpDependencyTree +{ + my ($ua, $tree) = @_; + # + # TODO: this could fail, how to properly promisify? + # + my $errors = $tree->{errors} // {}; + my $errorCount = $errors->{errors} // 0; + + if ($errorCount != 0) { + say "Unable to resolve dependencies, number of errors encountered is: $errorCount"; + my $p = Mojo::Promise->new(); + return $p->resolve(1); + } + + my $data = $tree->{data}; + if (!defined($data)) { + say "Unable to resolve dependencies, did not obtain (valid) results"; + my $p = Mojo::Promise->new(); + return $p->resolve(1); + } + else { + # + # TODO: this is *not* how we should await things in general. + # Fix using Mojo::AsyncAwait? + # + return _fetchModuleList($ua, '/modulesFromCommand')->then(sub { + my $list = shift; + my @names = map { $_->{name} } (@$list); + return @names; + })->then(sub { + my @modules = @_; + my $err = ksb::UserInterface::DependencyGraph::printTrees( + $data, + @modules + ); + return $err ? 1 : 0; + }); + } +} + # Returns a promise chain to handle the "debug and show some output but don't # actually build anything" use case. sub _runModeDebug { my $self = shift; my $app = $self->app; my $ua = $self->ua; my %debugFlags = %{$app->ksb->{debugFlags}}; $app->log->debug("Run mode: DEBUG"); if ($debugFlags{'dependency-tree'}) { $app->log->debug("Dumping dependency tree (in a later release...)"); return $ua->get_p('/moduleGraph')->then(sub { my $tx = _check_error(shift); - say $tx->result->text; - return 0; + return $tx->result->json; + })->then(sub { + my $tree = shift; + return dumpDependencyTree($ua, $tree); }); } elsif ($debugFlags{'list-build'}) { $app->log->debug("Listing modules to build"); return $ua->get_p('/modules')->then(sub { my $tx = _check_error(shift); my @modules = @{$tx->result->json}; say $_ foreach @modules; return 0; }); } return 0; # Bail early } # Returns a promise chain to handle the normal build case. sub _runModeBuild { my $self = shift; my $module_failures_ref = shift; my $ui = $self->ui; my $ua = $self->ua; my $app = $self->app; $app->log->debug("Run mode: BUILD"); # Open a file to log the event stream my $ctx = $app->context(); my $separator = ' '; my $dest = pretending() ? '/dev/null' : $ctx->getLogDirFor($ctx) . '/event-stream'; open my $event_stream, '>', $dest or croak_internal("Unable to open event log $!"); $event_stream->say("["); # Try to make it valid JSON syntax # We track the build using a JSON-based event stream which is published as # a WebSocket IPC using Mojolicious. We need to return a promise which # ultimately resolves to the exit status of the build. return $ua->websocket_p('/events')->then(sub { # Websocket Event handler my $ws = shift; my $everFailed = 0; my $stop_promise = Mojo::Promise->new; # Websockets seem to be inherently event-driven instead of simply # client/server. So attach the event handlers and then return to the event # loop to await progress. $ws->on(json => sub { # This handler is called by the backend when there is something notable # to report my ($ws, $resultRef) = @_; foreach my $modRef (@{$resultRef}) { # Update the U/I eval { $ui->notifyEvent($modRef); $event_stream->say($separator . to_json($modRef)); $separator = ', '; }; if ($@) { $ws->finish; $stop_promise->reject($@); } # See ksb::StatusMonitor for where events defined if ($modRef->{event} eq 'phase_completed') { my $results = $modRef->{phase_completed}; push @{$module_failures_ref}, $results if $results->{result} eq 'error'; } if ($modRef->{event} eq 'build_done') { # We've reported the build is complete, activate the promise # holding things together. The value we pass is what is passed # to the next promise handler. $stop_promise->resolve(scalar @{$module_failures_ref}); } } }); $ws->on(finish => sub { # Shouldn't happen in a normal build but it's probably possible $stop_promise->reject; # ignored if we resolved first }); # Blocking call to kick off the build my $tx = $ua->post('/build'); if (my $err = $tx->error) { $stop_promise->reject('Unable to start build: ' . $err->{message}); } # Once we return here we'll wait in Mojolicious event loop for awhile until # the build is done, before moving into the promise handler below return $stop_promise; })->finally(sub { $event_stream->say("]"); $event_stream->close(); }); } # Just a giant huge promise handler that actually processes U/I events and # keeps the TTY up to date. Note the TTY-specific stuff is actually itself # buried in a separate class for now. sub start { my $self = shift; my $ua = $self->ua; my $app = $self->app; my $result = 0; # notes errors from module builds or internal errors my @module_failures; $app->log->debug("Sending test msg to backend"); # This call just reads an option from the BuildContext as a sanity check $ua->get_p('/context/options/pretend')->then(sub { my $tx = shift; _check_error($tx); # If we get here things are mostly working? my $selectorsRef = $app->{selectors}; # We need to specifically ask for all modules if we're not passing a # specific list of modules to build. my $headers = { }; $headers->{'X-BuildAllModules'} = 1 unless @{$selectorsRef}; $app->log->debug("Test msg success, sending selectors to build"); # Tell the backend which modules to build. return $ua->post_p('/modules', $headers, json => $selectorsRef); })->then(sub { my $tx = shift; _check_error($tx); my $result = eval { $tx->result->json->[0]; }; $app->log->debug("Selectors sent to backend, $result"); # We've received a successful response from the backend that it's able to # build the requested modules, so proceed as appropriate based on the run mode # the user has requested. $app->ksb->{debugFlags} //= {}; return $self->_runModeDebug() if (%{$app->ksb->{debugFlags}}); return $self->_runModeBuild(\@module_failures); })->then(sub { # Build done, value comes from runMode promise above $result ||= shift; $app->log->debug("Chosen run mode complete, result (0 == success): $result"); })->catch(sub { # Catches all errors in any of the prior promises my $err = shift; say "Error: ", $err->{code}, " ", $err->{message}; # See if we made it to an rc-file my $ctx = $app->ksb->context(); my $rcFile = $ctx ? $ctx->rcFile() // 'Unknown' : undef; say "Using configuration file found at $rcFile" if $rcFile; $result = 1; # error })->wait; # _report_on_failures(@module_failures); return $result; }; sub _report_on_failures { my @failures = @_; my $max_width = max map { length ($_->{module}) } @failures; foreach my $mod (@failures) { my $module = $mod->{module}; my $phase = $mod->{phase}; my $log = $mod->{error_file}; my $padding = $max_width - length $module; $module .= (' ' x $padding); # Left-align $phase = 'setup buildsystem' if $phase eq 'buildsystem'; error("b[*] r[b[$module] failed to b[$phase]"); error("b[*]\tFind the log at file://$log") if $log; } } 1; diff --git a/modules/ksb/dto/ModuleInfo.pm b/modules/ksb/dto/ModuleInfo.pm new file mode 100644 index 0000000..34e4b7c --- /dev/null +++ b/modules/ksb/dto/ModuleInfo.pm @@ -0,0 +1,48 @@ +package ksb::dto::ModuleInfo; + +use strict; +use warnings; +use 5.014; + +use ksb::Module; + + +# +# dto::ModuleInfo +# +# This module provides utilities to convert from the internal representation +# of a module object to its wire format equivalent, and vice versa. +# Using DTOs is a well known good practice to ensure API output reflects a +# semantically meaningful view, without cluttering it with internal, +# implementation specific notions of what 'model' is actually represented. +# + +sub selectedModuleToDto +{ + my $graph = shift; + # TODO + # Perl WTF: does *not* work: assert_isa(shift, 'ksb::Module'); + # but say (ref $module) outputs ksb::Module + my $module = shift; + + my $name = $module->name(); + my $branch = $graph->{$name}->{branch} // ''; + my $isBuilt = $graph->{$name}->{build} ? 1: 0; + my $dto = { + name => $name, + path => $graph->{$name}->{path} + }; + + $dto->{branch} = "$branch" unless $branch eq ''; + $dto->{build} = \$isBuilt; + return $dto; +} + +sub selectedModulesToDtos +{ + my ($graph, $modules) = @_; + my @dtos = map { selectedModuleToDto($graph, $_); } (@$modules); + return @dtos; +} + +1; diff --git a/modules/web/BackendServer.pm b/modules/web/BackendServer.pm index 4951500..befd7be 100644 --- a/modules/web/BackendServer.pm +++ b/modules/web/BackendServer.pm @@ -1,274 +1,301 @@ package web::BackendServer; # Make this subclass a Mojolicious app use Mojo::Base 'Mojolicious'; use Mojo::Util qw(trim); use ksb::Application; use ksb::dto::ModuleGraph; +use ksb::dto::ModuleInfo; +use ksb::DependencyResolver; use Cwd; # This is written in a kind of domain-specific language for Mojolicious for # now, to setup a web server backend for clients / frontends to communicate # with. # See https://mojolicious.org/perldoc/Mojolicious/Guides/Tutorial has 'options'; has 'selectors'; sub new { my ($class, @opts) = @_; return $class->SUPER::new(options => [@opts], ksbhome => getcwd()); } # Adds a helper method to each HTTP context object to return the # ksb::Application class in use sub make_new_ksb { my $c = shift; # ksb::Application startup uses current dir to find right rc-file # by default. chdir($c->app->{ksbhome}); my $app = ksb::Application->new->setHeadless; my @selectors = $app->establishContext(@{$c->app->{options}}); $c->app->selectors([@selectors]); $c->app->log->info("Selectors are ", join(', ', @selectors)); return $app; } # Package-shared variables for helpers and closures my $LAST_RESULT; my $BUILD_PROMISE; my $IN_PROGRESS; my $KSB_APP; sub startup { my $self = shift; # Force use of 'modules/web' as the home directory, would normally be # 'modules' alone $self->home($self->home->child('web')); # Fixup templates and public base directories $self->static->paths->[0] = $self->home->child('public'); $self->renderer->paths->[0] = $self->home->child('templates'); $self->helper(ksb => sub { my ($c, $new_ksb) = @_; $KSB_APP = $new_ksb if $new_ksb; $KSB_APP //= make_new_ksb($c); return $KSB_APP; }); $self->helper(in_build => sub { $IN_PROGRESS }); $self->helper(context => sub { shift->ksb->context() }); my $r = $self->routes; $self->_generateRoutes; return; } sub _generateRoutes { my $self = shift; my $r = $self->routes; $r->get('/' => 'index'); $r->post('/reset' => sub { my $c = shift; if ($c->in_build || !defined $LAST_RESULT) { $c->res->code(400); return $c->render; } my $old_result = $LAST_RESULT; $c->ksb(make_new_ksb($c)); undef $LAST_RESULT; $c->render(json => { last_result => $old_result }); }); $r->get('/context/options' => sub { my $c = shift; $c->render(json => $c->ksb->context()->{options}); }); $r->get('/context/options/:option' => sub { my $c = shift; my $ctx = $c->ksb->context(); my $opt = $c->param('option') or do { $c->res->code(400); return $c->render; }; if (defined $ctx->{options}->{$opt}) { $c->render(json => { $opt => $ctx->{options}->{$opt} }); } else { $c->res->code(404); $c->reply->not_found; } }); $r->get('/modules' => sub { my $c = shift; $c->render(json => $c->ksb->context()->moduleList()); } => 'module_lookup'); $r->get('/known_modules' => sub { my $c = shift; my $resolver = $c->ksb->{module_resolver}; my @setsAndModules = @{$resolver->{inputModulesAndOptions}}; my @output = map { $_->isa('ksb::ModuleSet') ? [ $_->name(), $_->moduleNamesToFind() ] : $_->name() # should be a ksb::Module } @setsAndModules; $c->render(json => \@output); }); $r->post('/modules' => sub { my $c = shift; my $selectorList = $c->req->json; my $build_all = $c->req->headers->header('X-BuildAllModules'); # Remove empty selectors my @selectors = grep { !!$_ } map { trim($_ // '') } @{$selectorList}; # If not building all then ensure there's at least one module to build if ($c->in_build || !$selectorList || (!@selectors && !$build_all) || (@selectors && $build_all)) { $c->app->log->error("Something was wrong with modules to assign to build"); return $c->render(text => "Invalid request sent", status => 400); } eval { my $workload = $c->ksb->modulesFromSelectors(@selectors); $c->ksb->setModulesToProcess($workload); }; if ($@) { return $c->render(text => $@->{message}, status => 400); } my $numSels = scalar @selectors; $c->render(json => ["$numSels handled"]); }, 'post_modules'); $r->get('/module/:modname' => sub { my $c = shift; my $name = $c->stash('modname'); my $module = $c->ksb->context()->lookupModule($name); if (!$module) { $c->render(template => 'does_not_exist'); return; } my $opts = { options => $module->{options}, persistent => $c->ksb->context()->{persistent_options}->{$name}, }; $c->render(json => $opts); }); $r->get('/module/:modname/logs/error' => sub { my $c = shift; my $name = $c->stash('modname'); $c->render(text => "TODO: Error logs for $name"); }); $r->get('/config' => sub { my $c = shift; $c->render(text => $c->ksb->context()->rcFile()); }); $r->post('/config' => sub { # TODO If new filename can be loaded, load it and reset application object die "Unimplemented"; }); $r->get('/build-metadata' => sub { die "Unimplemented"; }); $r->websocket('/events' => sub { my $c = shift; $c->inactivity_timeout(0); my $ctx = $c->ksb->context(); my $monitor = $ctx->statusMonitor(); # Send prior events the receiver wouldn't have received yet my @curEvents = $monitor->events(); $c->send({json => \@curEvents}); # Hook up an event handler to send future events as they're generated $monitor->on(newEvent => sub { my ($monitor, $resultRef) = @_; $c->on(drain => sub { $c->finish }) if ($resultRef->{event} eq 'build_done'); $c->send({json => [ $resultRef ]}); }); }); $r->get('/event_viewer' => sub { my $c = shift; $c->render(template => 'event_viewer'); }); $r->get('/building' => sub { my $c = shift; $c->render(text => $c->in_build ? 'True' : 'False'); }); $r->get('/moduleGraph' => sub { my $c = shift; my $work = $c->app->ksb->workLoad() // {}; my $info = $work->{dependencyInfo}; if (defined($info)) { my $dto = ksb::dto::ModuleGraph::dependencyInfoToDto($info); $c->render(json => $dto); } else { $c->reply->not_found; } }); + $r->get('/modulesFromCommand' => sub { + my $c = shift; + my $work = $c->app->ksb->workLoad() // {}; + my $info = $work->{dependencyInfo}; + + if (defined($info) + && !ksb::DependencyResolver::hasErrors($info) + && exists $info->{graph}) { + my $graph = $info->{graph}; + my $modules = $work->{modulesFromCommand}; + my @dtos = ksb::dto::ModuleInfo::selectedModulesToDtos( + $graph, + $modules + ); + # + # Trap for the unwary: make sure to return a reference. + # Without this Mojolicious won't encode the array properly + # + $c->render(json => \@dtos); + } + else { + $c->reply->not_found; + } + }); + $r->post('/build' => sub { my $c = shift; if ($c->in_build) { $c->res->code(400); $c->render(text => 'Build already in progress, cancel it first.'); return; } $c->app->log->debug('Starting build'); $IN_PROGRESS = 1; $BUILD_PROMISE = $c->ksb->startHeadlessBuild->finally(sub { my ($result) = @_; $c->app->log->debug("Build done"); $IN_PROGRESS = 0; return $LAST_RESULT = $result; }); $c->render(text => $c->url_for('event_viewer')->to_abs->to_string); }); } 1;