diff --git a/modules/ksb/DependencyResolver.pm b/modules/ksb/DependencyResolver.pm index bebd7cd..28841d7 100644 --- a/modules/ksb/DependencyResolver.pm +++ b/modules/ksb/DependencyResolver.pm @@ -1,838 +1,768 @@ package ksb::DependencyResolver; # Class: DependencyResolver # # This module handles resolving dependencies between modules. Each "module" # from the perspective of this resolver is simply a module full name, as # given by the KDE Project database. (e.g. extragear/utils/kdesrc-build) use strict; use warnings; use 5.014; our $VERSION = '0.20'; use ksb::BuildException; use ksb::Debug; use ksb::Util; use List::Util qw(first); sub uniq { my %seen; return grep { ++($seen{$_}) == 1 } @_; } # Constructor: new # # Constructs a new . # # Parameters: # # moduleFactoryRef - Reference to a sub that creates ksb::Modules from # kde-project module names. Used for ksb::Modules for which the user # requested recursive dependency inclusion. # # Synposis: # # > my $resolver = new DependencyResolver($modNewRef); # > $resolver->readDependencyData(open my $fh, '<', 'file.txt'); # > $resolver->resolveDependencies(@modules); sub new { my $class = shift; my $moduleFactoryRef = shift; my $self = { # hash table mapping short module names (m) to a hashref key by branch # name, the value of which is yet another hashref (see # readDependencyData). Note that this assumes KDE git infrastructure # ensures that all full module names (e.g. # kde/workspace/plasma-workspace) map to a *unique* short name (e.g. # plasma-workspace) by stripping leading path components dependenciesOf => { }, # hash table mapping a wildcarded module name with no branch to a # listref of module:branch dependencies. catchAllDependencies => { }, # reference to a sub that will properly create a ksb::Module from a # given kde-project module name. Used to support automatically adding # dependencies to a build. moduleFactoryRef => $moduleFactoryRef, }; return bless $self, $class; } # Function: shortenModuleName # # Internal: # # This method returns the 'short' module name of kde-project full project paths. # E.g. 'kde/kdelibs/foo' would be shortened to 'foo'. # # This is a static function, not an object method. # # Parameters: # # path - A string holding the full module virtual path # # Returns: # # The module name. sub _shortenModuleName { my $name = shift; $name =~ s{^.*/}{}; # Uses greedy capture by default return $name; } # Method: readDependencyData # # Reads in dependency data in a pseudo-Makefile format. # See kde-build-metadata/dependency-data. # # Parameters: # $self - The DependencyResolver object. # $fh - Filehandle to read dependencies from (should already be open). # # Exceptions: # Can throw an exception on I/O errors or malformed dependencies. sub readDependencyData { my $self = assert_isa(shift, 'ksb::DependencyResolver'); my $fh = shift; my $dependenciesOfRef = $self->{dependenciesOf}; my $dependencyAtom = qr/ ^\s* # Clear leading whitespace ([^\[:\s]+) # (1) Capture anything not a [, :, or whitespace (dependent item) \s* # Clear whitespace we didn't capture (?:\[ # Open a non-capture group... ([^\]:\s]+) # (2) Capture branch name without brackets ])?+ # Close group, make optional, no backtracking \s* # Clear whitespace we didn't capture : \s* ([^\s\[]+) # (3) Capture all non-whitespace (source item) (?:\s*\[ # Open a non-capture group... ([^\]\s]+) # (4) Capture branch name without brackets ])?+ # Close group, make optional, no backtracking \s*$ # Ensure no trailing cruft. Any whitespace should end line /x; # /x Enables extended whitespace mode while(my $line = <$fh>) { # Strip comments, skip empty lines. $line =~ s{#.*$}{}; next if $line =~ /^\s*$/; if ($line !~ $dependencyAtom) { croak_internal("Invalid line $line when reading dependency data."); } my ($dependentItem, $dependentBranch, $sourceItem, $sourceBranch) = $line =~ $dependencyAtom; # Ignore "catch-all" dependencies where the source is the catch-all if ($sourceItem =~ m,\*$,) { warning ("\tIgnoring dependency on wildcard module grouping " . "on line $. of kde-build-metadata/dependency-data"); next; } $dependentBranch ||= '*'; # If no branch, apply catch-all flag $sourceBranch ||= '*'; # _shortenModuleName may remove negation marker so check now my $depKey = (index($sourceItem, '-') == 0) ? '-' : '+'; $sourceItem =~ s/^-//; # remove negation marker if name already short # Source can never be a catch-all so we can shorten early. Also, # we *must* shorten early to avoid a dependency on a long path. $sourceItem = _shortenModuleName($sourceItem); # Handle catch-all dependent groupings if ($dependentItem =~ /\*$/) { $self->{catchAllDependencies}->{$dependentItem} //= [ ]; push @{$self->{catchAllDependencies}->{$dependentItem}}, "$sourceItem:$sourceBranch"; next; } $dependentItem = _shortenModuleName($dependentItem); # Initialize with hashref if not already defined. The hashref will hold # - => [ ] (list of explicit *NON* dependencies of item:$branch), # + => [ ] (list of dependencies of item:$branch) # # Each dependency item is tracked at the module:branch level, and there # is always at least an entry for module:*, where '*' means branch # is unspecified and should only be used to add dependencies, never # take them away. # # Finally, all (non-)dependencies in a list are also of the form # fullname:branch, where "*" is a valid branch. $dependenciesOfRef->{"$dependentItem:*"} //= { '-' => [ ], '+' => [ ], }; # Create actual branch entry if not present $dependenciesOfRef->{"$dependentItem:$dependentBranch"} //= { '-' => [ ], '+' => [ ], }; push @{$dependenciesOfRef->{"$dependentItem:$dependentBranch"}->{$depKey}}, "$sourceItem:$sourceBranch"; } $self->_canonicalizeDependencies(); } # Function: _canonicalizeDependencies # # Ensures that all stored dependencies are stored in a way that allows for # reproducable dependency ordering (assuming the same dependency items and same # selectors are used). # # Parameters: none # # Returns: none sub _canonicalizeDependencies { my $self = shift; my $dependenciesOfRef = $self->{dependenciesOf}; foreach my $dependenciesRef (values %{$dependenciesOfRef}) { @{$dependenciesRef->{'-'}} = sort @{$dependenciesRef->{'-'}}; @{$dependenciesRef->{'+'}} = sort @{$dependenciesRef->{'+'}}; } } sub _lookupDirectDependencies { my $self = assert_isa(shift, 'ksb::DependencyResolver'); my ($path, $branch) = @_; my $dependenciesOfRef = $self->{dependenciesOf}; my @directDeps = (); my @exclusions = (); my $item = _shortenModuleName($path); my $moduleDepEntryRef = $dependenciesOfRef->{"$item:*"}; if ($moduleDepEntryRef) { debug("handling dependencies for: $item without branch (*)"); push @directDeps, @{$moduleDepEntryRef->{'+'}}; push @exclusions, @{$moduleDepEntryRef->{'-'}}; } if ($branch && $branch ne '*') { $moduleDepEntryRef = $dependenciesOfRef->{"$item:$branch"}; if ($moduleDepEntryRef) { debug("handling dependencies for: $item with branch ($branch)"); push @directDeps, @{$moduleDepEntryRef->{'+'}}; push @exclusions, @{$moduleDepEntryRef->{'-'}}; } } while (my ($catchAll, $deps) = each %{$self->{catchAllDependencies}}) { my $prefix = $catchAll; $prefix =~ s/\*$//; if (($path =~ /^$prefix/) || !$prefix) { push @directDeps, @{$deps}; } } foreach my $exclusion (@exclusions) { # Remove only modules at the exact given branch as a dep. # However catch-alls can remove catch-alls. # But catch-alls cannot remove a specific branch, such exclusions have # to also be specific. @directDeps = grep { $_ ne $exclusion } (@directDeps); } my $result = { syntaxErrors => 0, trivialCycles => 0, dependencies => {} }; for my $dep (@directDeps) { my ($depPath, $depBranch) = ($dep =~ m/^([^:]+):(.*)$/); if (!$depPath) { error("r[Invalid dependency declaration: b[$dep]]"); ++($result->{syntaxErrors}); next; } my $depItem = _shortenModuleName($depPath); if ($depItem eq $item) { debug("\tBreaking trivial cycle of b[$depItem] -> b[$item]"); ++($result->{trivialCycles}); next; } if ($result->{dependencies}->{$depItem}) { debug("\tSkipping duplicate direct dependency b[$depItem] of b[$item]"); } else { $depBranch //= ''; # work-around: wildcard branches are a don't care, not an actual # branch name/value $depBranch = undef if ($depBranch eq '' || $depBranch eq '*'); $result->{dependencies}->{$depItem} = { item => $depItem, path => $depPath, branch => $depBranch }; } } return $result; } sub _runDependencyVote { my $moduleGraph = shift; for my $item (keys(%$moduleGraph)) { my @names = keys(%{$moduleGraph->{$item}->{allDeps}->{items}}); for my $name (@names) { ++($moduleGraph->{$name}->{votes}->{$item}); } } return $moduleGraph; } sub _detectDependencyCycle { my ($moduleGraph, $depItem, $item) = @_; my $depModuleGraph = $moduleGraph->{$depItem}; if ($depModuleGraph->{traces}->{status}) { if ($depModuleGraph->{traces}->{status} == 2) { debug("Already resolved $depItem -- skipping"); return $depModuleGraph->{traces}->{result}; } else { error("Found a dependency cycle at: $depItem while tracing $item"); $depModuleGraph->{traces}->{result} = 1; } } else { $depModuleGraph->{traces}->{status} = 1; $depModuleGraph->{traces}->{result} = 0; my @names = keys(%{$depModuleGraph->{deps}}); for my $name (@names) { if (_detectDependencyCycle($moduleGraph, $name, $item)) { $depModuleGraph->{traces}->{result} = 1; } } } $depModuleGraph->{traces}->{status} = 2; return $depModuleGraph->{traces}->{result}; } sub _checkDependencyCycles { my $moduleGraph = shift; my $errors = 0; for my $item (keys(%$moduleGraph)) { if(_detectDependencyCycle($moduleGraph, $item, $item)) { error("Somehow there is a circular dependency involving b[$item]! :("); error("Please file a bug against kde-build-metadata about this!"); ++$errors; } } return $errors; } sub _copyUpDependenciesForModule { my ($moduleGraph, $item) = @_; my $allDeps = $moduleGraph->{$item}->{allDeps}; if($allDeps->{done}) { debug("\tAlready copied up dependencies for b[$item] -- skipping"); } else { debug("\tCopying up dependencies and transitive dependencies for item: b[$item]"); $allDeps->{items} = {}; my @names = keys(%{$moduleGraph->{$item}->{deps}}); for my $name (@names) { if ($allDeps->{items}->{$name}) { debug("\tAlready copied up (transitive) dependency on b[$name] for b[$item] -- skipping"); } else { _copyUpDependenciesForModule($moduleGraph, $name); my @copied = keys(%{$moduleGraph->{$name}->{allDeps}->{items}}); for my $copy (@copied) { if ($allDeps->{items}->{$copy}) { debug("\tAlready copied up (transitive) dependency on b[$copy] for b[$item] -- skipping"); } else { ++($allDeps->{items}->{$copy}); } } ++($allDeps->{items}->{$name}); } } ++($allDeps->{done}); } } sub _copyUpDependencies { my $moduleGraph = shift; for my $item (keys(%$moduleGraph)) { _copyUpDependenciesForModule($moduleGraph, $item); } return $moduleGraph; } sub _detectBranchConflict { my ($moduleGraph, $item, $branch) = @_; if ($branch) { my $subGraph = $moduleGraph->{$item}; my $previouslySelectedBranch = $subGraph->{branch}; return $previouslySelectedBranch if($previouslySelectedBranch && $previouslySelectedBranch ne $branch); } return undef; } sub _getDependencyPathOf { my ($module, $item, $path) = @_; if ($module) { my $projectPath = $module->fullProjectPath(); $projectPath = "third-party/$projectPath" if(!$module->isKDEProject()); debug("\tUsing path: 'b[$projectPath]' for item: b[$item]"); return $projectPath; } debug("\tGuessing path: 'b[$path]' for item: b[$item]"); return $path; } sub _resolveDependenciesForModuleDescription { my $self = assert_isa(shift, 'ksb::DependencyResolver'); my ($moduleGraph, $moduleDesc) = @_; my $module = $moduleDesc->{module}; if($module) { assert_isa($module, 'ksb::Module'); } my $path = $moduleDesc->{path}; my $item = $moduleDesc->{item}; my $branch = $moduleDesc->{branch}; my $prettyBranch = $branch ? "$branch" : "*"; my $includeDependencies = $module ? $module->getOption('include-dependencies') : $moduleDesc->{includeDependencies}; my $errors = { syntaxErrors => 0, trivialCycles => 0, branchErrors => 0 }; debug("Resolving dependencies for module: b[$item]"); while (my ($depItem, $depInfo) = each %{$moduleGraph->{$item}->{deps}}) { my $depPath = $depInfo->{path}; my $depBranch = $depInfo->{branch}; my $prettyDepBranch = $depBranch ? "$depBranch" : "*"; debug ("\tdep-resolv: b[$item:$prettyBranch] depends on b[$depItem:$prettyDepBranch]"); my $depModuleGraph = $moduleGraph->{$depItem}; if($depModuleGraph) { my $previouslySelectedBranch = _detectBranchConflict($moduleGraph, $depItem, $depBranch); if($previouslySelectedBranch) { error("r[Found a dependency conflict in branches ('b[$previouslySelectedBranch]' is not 'b[$prettyDepBranch]') for b[$depItem]! :("); ++($errors->{branchErrors}); } else { if($depBranch) { $depModuleGraph->{branch} = $depBranch; } } } else { my $depModule = $self->{moduleFactoryRef}($depItem); my $resolvedPath = _getDependencyPathOf($depModule, $depItem, $depPath); # May not exist, e.g. misspellings or 'virtual' dependencies like kf5umbrella. if(!$depModule) { debug("\tdep-resolve: Will not build virtual or undefined module: b[$depItem]\n"); } my $depLookupResult = $self->_lookupDirectDependencies( $resolvedPath, $depBranch ); $errors->{trivialCycles} += $depLookupResult->{trivialCycles}; $errors->{syntaxErrors} += $depLookupResult->{syntaxErrors}; $moduleGraph->{$depItem} = { votes => {}, path => $resolvedPath, build => $depModule && $includeDependencies ? 1 : 0, branch => $depBranch, deps => $depLookupResult->{dependencies}, allDeps => {}, module => $depModule, traces => {} }; my $depModuleDesc = { includeDependencies => $includeDependencies, module => $depModule, item => $depItem, path => $resolvedPath, branch => $depBranch }; if (!$moduleGraph->{$depItem}->{build}) { debug (" y[b[*] $item depends on $depItem, but no module builds $depItem for this run.]"); } if($depModule && $depBranch && (_getBranchOf($depModule) // '') ne "$depBranch") { my $wrongBranch = _getBranchOf($depModule) // '?'; error(" r[b[*] $item needs $depItem:$prettyDepBranch, not $depItem:$wrongBranch]"); ++($errors->{branchErrors}); } debug("Resolving transitive dependencies for module: b[$item] (via: b[$depItem:$prettyDepBranch])"); my $resolvErrors = $self->_resolveDependenciesForModuleDescription( $moduleGraph, $depModuleDesc ); $errors->{branchErrors} += $resolvErrors->{branchErrors}; $errors->{syntaxErrors} += $resolvErrors->{syntaxErrors}; $errors->{trivialCycles} += $resolvErrors->{trivialCycles}; } } return $errors; } sub resolveToModuleGraph { my $self = assert_isa(shift, 'ksb::DependencyResolver'); my @modules = @_; my %graph; my $moduleGraph = \%graph; my $result = { graph => $moduleGraph, errors => { branchErrors => 0, pathErrors => 0, trivialCycles => 0, syntaxErrors => 0, cycles => 0 } }; my $errors = $result->{errors}; for my $module (@modules) { my $item = $module->name(); # _shortenModuleName($path); my $branch = _getBranchOf($module); my $path = _getDependencyPathOf($module, $item, ''); if (!$path) { error("r[Unable to determine project/dependency path of module: $item]"); ++($errors->{pathErrors}); next; } if($moduleGraph->{$item}) { debug("Module pulled in previously through (transitive) dependencies: $item"); my $previouslySelectedBranch = _detectBranchConflict($moduleGraph, $item, $branch); if($previouslySelectedBranch) { error("r[Found a dependency conflict in branches ('b[$previouslySelectedBranch]' is not 'b[$branch]') for b[$item]! :("); ++($errors->{branchErrors}); } elsif ($branch) { $moduleGraph->{$item}->{branch} = $branch; } # # May have been pulled in via dependencies but not yet marked for # build. Do so now, since it is listed explicitly in @modules # $moduleGraph->{$item}->{build} = 1; } else { my $depLookupResult = $self->_lookupDirectDependencies( $path, $branch ); $errors->{trivialCycles} += $depLookupResult->{trivialCycles}; $errors->{syntaxErrors} += $depLookupResult->{syntaxErrors}; $moduleGraph->{$item} = { votes => {}, path => $path, build => 1, branch => $branch, module => $module, deps => $depLookupResult->{dependencies}, allDeps => {}, traces => {} }; my $moduleDesc = { includeDependencies => $module->getOption('include-dependencies'), path => $path, item => $item, branch => $branch, module => $module }; my $resolvErrors = $self->_resolveDependenciesForModuleDescription( $moduleGraph, $moduleDesc ); $errors->{branchErrors} += $resolvErrors->{branchErrors}; $errors->{syntaxErrors} += $resolvErrors->{syntaxErrors}; $errors->{trivialCycles} += $resolvErrors->{trivialCycles}; } } my $pathErrors = $errors->{pathErrors}; if ($pathErrors) { error("Total of items which were not resolved due to path lookup failure: $pathErrors"); } my $branchErrors = $errors->{branchErrors}; if ($branchErrors) { error("Total of branch conflicts detected: $branchErrors"); } my $syntaxErrors = $errors->{syntaxErrors}; if ($syntaxErrors) { error("Total of encountered syntax errors: $syntaxErrors"); } if ($syntaxErrors || $pathErrors || $branchErrors) { error("Unable to resolve dependency graph"); $result->{graph} = undef; return $result; } my $trivialCycles = $errors->{trivialCycles}; if ($trivialCycles) { warning("Total of 'trivial' dependency cycles detected & eliminated: $trivialCycles"); } my $cycles = _checkDependencyCycles($moduleGraph); if ($cycles) { error("Total of items with at least one circular dependency detected: $errors"); error("Unable to resolve dependency graph"); $result->{cycles} = $cycles; $result->{graph} = undef; return $result; } else { $result->{graph} = _runDependencyVote(_copyUpDependencies($moduleGraph)); return $result; } } -sub _descendModuleGraph -{ - my ($moduleGraph, $callback, $nodeInfo, $context) = @_; - - my $depth = $nodeInfo->{depth}; - my $index = $nodeInfo->{idx}; - my $count = $nodeInfo->{count}; - my $currentItem = $nodeInfo->{currentItem}; - my $currentBranch = $nodeInfo->{currentBranch}; - my $parentItem = $nodeInfo->{parentItem}; - my $parentBranch = $nodeInfo->{parentBranch}; - - my $subGraph = $moduleGraph->{$currentItem}; - &$callback($nodeInfo, $subGraph->{module}, $context); - - ++$depth; - - my @items = keys(%{$subGraph->{deps}}); - - my $itemCount = scalar(@items); - my $itemIndex = 1; - - for my $item (@items) - { - $subGraph = $moduleGraph->{$item}; - my $branch = $subGraph->{branch} // ''; - my $itemInfo = { - build => $subGraph->{build}, - depth => $depth, - idx => $itemIndex, - count => $itemCount, - currentItem => $item, - currentBranch => $branch, - parentItem => $currentItem, - parentBranch => $currentBranch - }; - _descendModuleGraph($moduleGraph, $callback, $itemInfo, $context); - ++$itemIndex; - } -} - -sub walkModuleDependencyTrees -{ - my $moduleGraph = shift; - my $callback = shift; - my $context = shift; - my @modules = @_; - my $itemCount = scalar(@modules); - my $itemIndex = 1; - - for my $module (@modules) { - assert_isa($module, 'ksb::Module'); - my $item = $module->name(); - my $subGraph = $moduleGraph->{$item}; - my $branch = $subGraph->{branch} // ''; - my $info = { - build => $subGraph->{build}, - depth => 0, - idx => $itemIndex, - count => $itemCount, - currentItem => $item, - currentBranch => $branch, - parentItem => '', - parentBranch => '' - }; - _descendModuleGraph($moduleGraph, $callback, $info, $context); - ++$itemIndex; - } -} - sub hasErrors { my $info = shift; my $cycles = $info->{cycles} // 0; my $pathErrors = $info->{pathErrors} // 0; my $branchErrors = $info->{branchErrors} // 0; my $syntaxErrors = $info->{syntaxErrors} // 0; return $cycles || $pathErrors || $branchErrors || $syntaxErrors; } sub _compareBuildOrder { my ($moduleGraph, $a, $b) = @_; # # Enforce a strict dependency ordering. # The case where both are true should never happen, since that would # amount to a cycle, and cycle detection is supposed to have been # performed beforehand. # my $bDependsOnA = $moduleGraph->{$a}->{votes}->{$b} // 0; my $aDependsOnB = $moduleGraph->{$b}->{votes}->{$a} // 0; my $order = $bDependsOnA ? -1 : ($aDependsOnB ? 1 : 0); return $order if $order; # # Assuming no dependency relation, next sort by 'popularity': # the item with the most votes (back edges) is depended on the most # so it is probably a good idea to build that one earlier to help # maximise the duration of time for which builds can be run in parallel # my $voteA = scalar keys %{$moduleGraph->{$a}->{votes}}; my $voteB = scalar keys %{$moduleGraph->{$b}->{votes}}; my $votes = $voteB <=> $voteA; return $votes if $votes; # # If there is no good reason to perfer one module over another, # simply sort by name to get a reproducible build order. # That simplifies autotesting and/or reproducible builds. # (The items to sort are supplied as a hash so the order of keys is by # definition not guaranteed.) # my $name = ($a cmp $b); return $name; } sub sortModulesIntoBuildOrder { my $moduleGraph = shift; my @resolved = keys(%{$moduleGraph}); my @built = grep { $moduleGraph->{$_}->{build} && $moduleGraph->{$_}->{module} } (@resolved); my @prioritised = sort { _compareBuildOrder($moduleGraph, $a, $b); } (@built); my @modules = map { $moduleGraph->{$_}->{module} } (@prioritised); return @modules; } # Function: getBranchOf # # Internal: # # This function extracts the branch of the given Module by calling its # scm object's branch-determining method. It also ensures that the branch # returned was really intended to be a branch (as opposed to a detached HEAD); # undef is returned when the desired commit is not a branch name, otherwise # the user-requested branch name is returned. sub _getBranchOf { my $module = shift; my $scm = $module->scm(); # when the module's SCM is not git, # assume the default "no particular" branch wildcard return undef unless $scm->isa('ksb::Updater::Git'); my ($branch, $type) = $scm->_determinePreferredCheckoutSource($module); return ($type eq 'branch' ? $branch : undef); } 1; diff --git a/modules/ksb/UserInterface/DependencyGraph.pm b/modules/ksb/UserInterface/DependencyGraph.pm new file mode 100644 index 0000000..9270630 --- /dev/null +++ b/modules/ksb/UserInterface/DependencyGraph.pm @@ -0,0 +1,153 @@ +package ksb::UserInterface::DependencyGraph; + +use strict; +use warnings; +use 5.014; + +sub _descendModuleGraph +{ + my ($moduleGraph, $callback, $nodeInfo, $context) = @_; + + my $depth = $nodeInfo->{depth}; + my $index = $nodeInfo->{idx}; + my $count = $nodeInfo->{count}; + my $currentItem = $nodeInfo->{currentItem}; + my $currentBranch = $nodeInfo->{currentBranch}; + my $parentItem = $nodeInfo->{parentItem}; + my $parentBranch = $nodeInfo->{parentBranch}; + + my $subGraph = $moduleGraph->{$currentItem}; + &$callback($nodeInfo, $subGraph->{module}, $context); + + ++$depth; + + my @items = @{$subGraph->{deps}}; + + my $itemCount = scalar(@items); + my $itemIndex = 1; + + for my $item (@items) + { + $subGraph = $moduleGraph->{$item}; + my $branch = $subGraph->{branch} // ''; + my $itemInfo = { + build => $subGraph->{build}, + depth => $depth, + idx => $itemIndex, + count => $itemCount, + currentItem => $item, + currentBranch => $branch, + parentItem => $currentItem, + parentBranch => $currentBranch + }; + _descendModuleGraph($moduleGraph, $callback, $itemInfo, $context); + ++$itemIndex; + } +} + +sub _walkModuleDependencyTrees +{ + my $moduleGraph = shift; + my $callback = shift; + my $context = shift; + my @modules = @_; + my $itemCount = scalar(@modules); + my $itemIndex = 1; + + for my $item (@modules) { + my $subGraph = $moduleGraph->{$item}; + my $branch = $subGraph->{branch} // ''; + my $info = { + build => $subGraph->{build}, + depth => 0, + idx => $itemIndex, + count => $itemCount, + currentItem => $item, + currentBranch => $branch, + parentItem => '', + parentBranch => '' + }; + _descendModuleGraph($moduleGraph, $callback, $info, $context); + ++$itemIndex; + } +} + + +sub _treeOutputConnectors +{ + my ($depth, $index, $count) = @_; + my $blankPadding = (' ' x 4); + + return (' ── ', $blankPadding) if ($depth == 0); + return ('└── ', $blankPadding) if ($index == $count); + return ('├── ', '│ '); +} + +sub _yieldModuleDependencyTreeEntry +{ + my ($nodeInfo, $module, $context) = @_; + + my $depth = $nodeInfo->{depth}; + my $index = $nodeInfo->{idx}; + my $count = $nodeInfo->{count}; + my $build = $nodeInfo->{build}; + my $currentItem = $nodeInfo->{currentItem}; + my $currentBranch = $nodeInfo->{currentBranch}; + my $parentItem = $nodeInfo->{parentItem}; + my $parentBranch = $nodeInfo->{parentBranch}; + + my $buildStatus = $build ? 'built' : 'not built'; + my $statusInfo = $currentBranch ? "($buildStatus: $currentBranch)" : "($buildStatus)"; + + my $connectorStack = $context->{stack}; + my $prefix = pop(@$connectorStack); + + while($context->{depth} > $depth) { + $prefix = pop(@$connectorStack); + --($context->{depth}); + } + + push(@$connectorStack, $prefix); + + my ($connector, $padding) = _treeOutputConnectors($depth, $index, $count); + + push(@$connectorStack, $prefix . $padding); + $context->{depth} = $depth + 1; + + my $line = $prefix . $connector . $currentItem . ' ' . $statusInfo; + $context->{report}($line); +} + +sub printTrees +{ + my $tree = shift; + my @modules = @_; + + # + # Hack: reopen STDOUT to get rid of ... "does not map to ascii" noise + # Yes: the code points do not map to ASCII, that is sort of the point + # + my $ok = open my $fh, '>&', STDOUT; + return 1 unless $ok; + + my $depTreeCtx = { + stack => [''], + depth => 0, + report => sub { + my $line = shift; + print $fh $line, "\n"; + } + }; + + _walkModuleDependencyTrees( + $tree, + \&_yieldModuleDependencyTreeEntry, + $depTreeCtx, + @modules + ); + + close $fh; + return 0; +} + +1; diff --git a/modules/ksb/UserInterface/TTY.pm b/modules/ksb/UserInterface/TTY.pm index edd1cc8..ae3bcc0 100755 --- a/modules/ksb/UserInterface/TTY.pm +++ b/modules/ksb/UserInterface/TTY.pm @@ -1,284 +1,339 @@ #!/usr/bin/env perl package ksb::UserInterface::TTY 0.10; =pod =head1 NAME ksb::UserInterface::TTY -- A command-line interface to the kdesrc-build backend =head1 DESCRIPTION This class is used to show a user interface for a kdesrc-build run at the command line (as opposed to a browser-based or GUI interface). Since the kdesrc-build backend is now meant to be headless and controlled via a Web-style API set (powered by Mojolicious), this class manages the interaction with that backend, also using Mojolicious to power the HTTP and WebSocket requests necessary. =head1 SYNOPSIS my $app = web::BackendServer->new(@ARGV); my $ui = ksb::UserInterface::TTY->new($app); exit $ui->start(); # Blocks! Returns a shell-style return code =cut use strict; use warnings; use 5.014; use Mojo::Base -base; use Mojo::Server::Daemon; use Mojo::IOLoop; use Mojo::UserAgent; use Mojo::JSON qw(to_json); use ksb::BuildException; use ksb::StatusView; use ksb::Util; use ksb::Debug; +use ksb::UserInterface::DependencyGraph; +use Mojo::Promise; use IO::Handle; # For methods on event_stream file use List::Util qw(max); has ua => sub { Mojo::UserAgent->new->inactivity_timeout(0) }; has ui => sub { ksb::StatusView->new() }; has 'app'; sub new { my ($class, $app) = @_; my $self = $class->SUPER::new(app => $app); # Mojo::UserAgent can be tied to a Mojolicious application server directly to # handle relative URLs, which is perfect for what we want. Making this # attachment will startup the Web server behind the scenes and allow $ua to # make HTTP requests. $self->ua->server->app($app); # $self->ua->server->app->log->level('debug'); $self->ua->server->app->log->level('fatal'); return $self; } sub _check_error { my $tx = shift; my $err = $tx->error or return $tx; my $body = $tx->res->body // ''; open my $fh, '<', \$body; my ($first_line) = <$fh> // ''; $err->{message} .= "\n$first_line" if $first_line; die $err; }; + +sub _fetchModuleList +{ + my ($ua, $list) = @_; + return $ua->get_p($list)->then(sub { + my $tx = _check_error(shift); + return $tx->result->json; + }); +} + +sub dumpDependencyTree +{ + my ($ua, $tree) = @_; + # + # TODO: this could fail, how to properly promisify? + # + my $errors = $tree->{errors} // {}; + my $errorCount = $errors->{errors} // 0; + + if ($errorCount != 0) { + say "Unable to resolve dependencies, number of errors encountered is: $errorCount"; + my $p = Mojo::Promise->new(); + return $p->resolve(1); + } + + my $data = $tree->{data}; + if (!defined($data)) { + say "Unable to resolve dependencies, did not obtain (valid) results"; + my $p = Mojo::Promise->new(); + return $p->resolve(1); + } + else { + # + # TODO: this is *not* how we should await things in general. + # Fix using Mojo::AsyncAwait? + # + return _fetchModuleList($ua, '/modulesFromCommand')->then(sub { + my $list = shift; + my @names = map { $_->{name} } (@$list); + return @names; + })->then(sub { + my @modules = @_; + my $err = ksb::UserInterface::DependencyGraph::printTrees( + $data, + @modules + ); + return $err ? 1 : 0; + }); + } +} + # Returns a promise chain to handle the "debug and show some output but don't # actually build anything" use case. sub _runModeDebug { my $self = shift; my $app = $self->app; my $ua = $self->ua; my %debugFlags = %{$app->ksb->{debugFlags}}; $app->log->debug("Run mode: DEBUG"); if ($debugFlags{'dependency-tree'}) { $app->log->debug("Dumping dependency tree (in a later release...)"); return $ua->get_p('/moduleGraph')->then(sub { my $tx = _check_error(shift); - say $tx->result->text; - return 0; + return $tx->result->json; + })->then(sub { + my $tree = shift; + return dumpDependencyTree($ua, $tree); }); } elsif ($debugFlags{'list-build'}) { $app->log->debug("Listing modules to build"); return $ua->get_p('/modules')->then(sub { my $tx = _check_error(shift); my @modules = @{$tx->result->json}; say $_ foreach @modules; return 0; }); } return 0; # Bail early } # Returns a promise chain to handle the normal build case. sub _runModeBuild { my $self = shift; my $module_failures_ref = shift; my $ui = $self->ui; my $ua = $self->ua; my $app = $self->app; $app->log->debug("Run mode: BUILD"); # Open a file to log the event stream my $ctx = $app->context(); my $separator = ' '; my $dest = pretending() ? '/dev/null' : $ctx->getLogDirFor($ctx) . '/event-stream'; open my $event_stream, '>', $dest or croak_internal("Unable to open event log $!"); $event_stream->say("["); # Try to make it valid JSON syntax # We track the build using a JSON-based event stream which is published as # a WebSocket IPC using Mojolicious. We need to return a promise which # ultimately resolves to the exit status of the build. return $ua->websocket_p('/events')->then(sub { # Websocket Event handler my $ws = shift; my $everFailed = 0; my $stop_promise = Mojo::Promise->new; # Websockets seem to be inherently event-driven instead of simply # client/server. So attach the event handlers and then return to the event # loop to await progress. $ws->on(json => sub { # This handler is called by the backend when there is something notable # to report my ($ws, $resultRef) = @_; foreach my $modRef (@{$resultRef}) { # Update the U/I eval { $ui->notifyEvent($modRef); $event_stream->say($separator . to_json($modRef)); $separator = ', '; }; if ($@) { $ws->finish; $stop_promise->reject($@); } # See ksb::StatusMonitor for where events defined if ($modRef->{event} eq 'phase_completed') { my $results = $modRef->{phase_completed}; push @{$module_failures_ref}, $results if $results->{result} eq 'error'; } if ($modRef->{event} eq 'build_done') { # We've reported the build is complete, activate the promise # holding things together. The value we pass is what is passed # to the next promise handler. $stop_promise->resolve(scalar @{$module_failures_ref}); } } }); $ws->on(finish => sub { # Shouldn't happen in a normal build but it's probably possible $stop_promise->reject; # ignored if we resolved first }); # Blocking call to kick off the build my $tx = $ua->post('/build'); if (my $err = $tx->error) { $stop_promise->reject('Unable to start build: ' . $err->{message}); } # Once we return here we'll wait in Mojolicious event loop for awhile until # the build is done, before moving into the promise handler below return $stop_promise; })->finally(sub { $event_stream->say("]"); $event_stream->close(); }); } # Just a giant huge promise handler that actually processes U/I events and # keeps the TTY up to date. Note the TTY-specific stuff is actually itself # buried in a separate class for now. sub start { my $self = shift; my $ua = $self->ua; my $app = $self->app; my $result = 0; # notes errors from module builds or internal errors my @module_failures; $app->log->debug("Sending test msg to backend"); # This call just reads an option from the BuildContext as a sanity check $ua->get_p('/context/options/pretend')->then(sub { my $tx = shift; _check_error($tx); # If we get here things are mostly working? my $selectorsRef = $app->{selectors}; # We need to specifically ask for all modules if we're not passing a # specific list of modules to build. my $headers = { }; $headers->{'X-BuildAllModules'} = 1 unless @{$selectorsRef}; $app->log->debug("Test msg success, sending selectors to build"); # Tell the backend which modules to build. return $ua->post_p('/modules', $headers, json => $selectorsRef); })->then(sub { my $tx = shift; _check_error($tx); my $result = eval { $tx->result->json->[0]; }; $app->log->debug("Selectors sent to backend, $result"); # We've received a successful response from the backend that it's able to # build the requested modules, so proceed as appropriate based on the run mode # the user has requested. $app->ksb->{debugFlags} //= {}; return $self->_runModeDebug() if (%{$app->ksb->{debugFlags}}); return $self->_runModeBuild(\@module_failures); })->then(sub { # Build done, value comes from runMode promise above $result ||= shift; $app->log->debug("Chosen run mode complete, result (0 == success): $result"); })->catch(sub { # Catches all errors in any of the prior promises my $err = shift; say "Error: ", $err->{code}, " ", $err->{message}; # See if we made it to an rc-file my $ctx = $app->ksb->context(); my $rcFile = $ctx ? $ctx->rcFile() // 'Unknown' : undef; say "Using configuration file found at $rcFile" if $rcFile; $result = 1; # error })->wait; # _report_on_failures(@module_failures); return $result; }; sub _report_on_failures { my @failures = @_; my $max_width = max map { length ($_->{module}) } @failures; foreach my $mod (@failures) { my $module = $mod->{module}; my $phase = $mod->{phase}; my $log = $mod->{error_file}; my $padding = $max_width - length $module; $module .= (' ' x $padding); # Left-align $phase = 'setup buildsystem' if $phase eq 'buildsystem'; error("b[*] r[b[$module] failed to b[$phase]"); error("b[*]\tFind the log at file://$log") if $log; } } 1;