diff --git a/modules/ksb/DependencyResolver.pm b/modules/ksb/DependencyResolver.pm index afc3927..25fd24a 100644 --- a/modules/ksb/DependencyResolver.pm +++ b/modules/ksb/DependencyResolver.pm @@ -1,542 +1,1061 @@ package ksb::DependencyResolver; # Class: DependencyResolver # # This module handles resolving dependencies between modules. Each "module" # from the perspective of this resolver is simply a module full name, as # given by the KDE Project database. (e.g. extragear/utils/kdesrc-build) use strict; use warnings; use 5.014; our $VERSION = '0.20'; use ksb::BuildException; use ksb::Debug; use ksb::Util; use List::Util qw(first); +sub uniq +{ + my %seen; + return grep { ++($seen{$_}) == 1 } @_; +} + # Constructor: new # # Constructs a new . # # Parameters: # # moduleFactoryRef - Reference to a sub that creates ksb::Modules from # kde-project module names. Used for ksb::Modules for which the user # requested recursive dependency inclusion. # # Synposis: # # > my $resolver = new DependencyResolver($modNewRef); # > $resolver->readDependencyData(open my $fh, '<', 'file.txt'); # > $resolver->resolveDependencies(@modules); sub new { my $class = shift; my $moduleFactoryRef = shift; my $self = { # hash table mapping short module names (m) to a hashref key by branch # name, the value of which is yet another hashref (see # readDependencyData). Note that this assumes KDE git infrastructure # ensures that all full module names (e.g. # kde/workspace/plasma-workspace) map to a *unique* short name (e.g. # plasma-workspace) by stripping leading path components dependenciesOf => { }, # hash table mapping a wildcarded module name with no branch to a # listref of module:branch dependencies. catchAllDependencies => { }, # reference to a sub that will properly create a ksb::Module from a # given kde-project module name. Used to support automatically adding # dependencies to a build. moduleFactoryRef => $moduleFactoryRef, }; return bless $self, $class; } # Function: shortenModuleName # # Internal: # # This method returns the 'short' module name of kde-project full project paths. # E.g. 'kde/kdelibs/foo' would be shortened to 'foo'. # # This is a static function, not an object method. # # Parameters: # # path - A string holding the full module virtual path # # Returns: # # The module name. sub _shortenModuleName { my $name = shift; $name =~ s{^.*/}{}; # Uses greedy capture by default return $name; } # Method: readDependencyData # # Reads in dependency data in a pseudo-Makefile format. # See kde-build-metadata/dependency-data. # # Parameters: # $self - The DependencyResolver object. # $fh - Filehandle to read dependencies from (should already be open). # # Exceptions: # Can throw an exception on I/O errors or malformed dependencies. sub readDependencyData { my $self = assert_isa(shift, 'ksb::DependencyResolver'); my $fh = shift; my $dependenciesOfRef = $self->{dependenciesOf}; my $dependencyAtom = qr/ ^\s* # Clear leading whitespace ([^\[:\s]+) # (1) Capture anything not a [, :, or whitespace (dependent item) \s* # Clear whitespace we didn't capture (?:\[ # Open a non-capture group... ([^\]:\s]+) # (2) Capture branch name without brackets ])?+ # Close group, make optional, no backtracking \s* # Clear whitespace we didn't capture : \s* ([^\s\[]+) # (3) Capture all non-whitespace (source item) (?:\s*\[ # Open a non-capture group... ([^\]\s]+) # (4) Capture branch name without brackets ])?+ # Close group, make optional, no backtracking \s*$ # Ensure no trailing cruft. Any whitespace should end line /x; # /x Enables extended whitespace mode while(my $line = <$fh>) { # Strip comments, skip empty lines. $line =~ s{#.*$}{}; next if $line =~ /^\s*$/; if ($line !~ $dependencyAtom) { croak_internal("Invalid line $line when reading dependency data."); } my ($dependentItem, $dependentBranch, $sourceItem, $sourceBranch) = $line =~ $dependencyAtom; # Ignore "catch-all" dependencies where the source is the catch-all if ($sourceItem =~ m,\*$,) { warning ("\tIgnoring dependency on wildcard module grouping " . "on line $. of kde-build-metadata/dependency-data"); next; } # Ignore deps on Qt, since we allow system Qt. next if $sourceItem =~ /^\s*Qt/ || $dependentItem =~ /^\s*Qt/; $dependentBranch ||= '*'; # If no branch, apply catch-all flag $sourceBranch ||= '*'; # Source can never be a catch-all so we can shorten early. Also, # we *must* shorten early to avoid a dependency on a long path. $sourceItem = _shortenModuleName($sourceItem); # Handle catch-all dependent groupings if ($dependentItem =~ /\*$/) { $self->{catchAllDependencies}->{$dependentItem} //= [ ]; push @{$self->{catchAllDependencies}->{$dependentItem}}, "$sourceItem:$sourceBranch"; next; } $dependentItem = _shortenModuleName($dependentItem); # Initialize with hashref if not already defined. The hashref will hold # - => [ ] (list of explicit *NON* dependencies of item:$branch), # + => [ ] (list of dependencies of item:$branch) # # Each dependency item is tracked at the module:branch level, and there # is always at least an entry for module:*, where '*' means branch # is unspecified and should only be used to add dependencies, never # take them away. # # Finally, all (non-)dependencies in a list are also of the form # fullname:branch, where "*" is a valid branch. $dependenciesOfRef->{"$dependentItem:*"} //= { '-' => [ ], '+' => [ ], }; # Create actual branch entry if not present $dependenciesOfRef->{"$dependentItem:$dependentBranch"} //= { '-' => [ ], '+' => [ ], }; my $depKey = (index($sourceItem, '-') == 0) ? '-' : '+'; $sourceItem =~ s/^-//; push @{$dependenciesOfRef->{"$dependentItem:$dependentBranch"}->{$depKey}}, "$sourceItem:$sourceBranch"; } $self->_canonicalizeDependencies(); } # Function: _canonicalizeDependencies # # Ensures that all stored dependencies are stored in a way that allows for # reproducable dependency ordering (assuming the same dependency items and same # selectors are used). # # Parameters: none # # Returns: none sub _canonicalizeDependencies { my $self = shift; my $dependenciesOfRef = $self->{dependenciesOf}; foreach my $dependenciesRef (values %{$dependenciesOfRef}) { @{$dependenciesRef->{'-'}} = sort @{$dependenciesRef->{'-'}}; @{$dependenciesRef->{'+'}} = sort @{$dependenciesRef->{'+'}}; } } # Function: directDependenciesOf # # Internal: # # Finds and returns the direct dependencies of the given module at a given # branch. This requires forming a list of dependencies for the module from the # "branch neutral" dependencies, adding branch-specific dependencies, and then # removing any explicit non-dependencies for the given branch, which is why # this is a separate routine. # # Parameters: # dependenciesOfRef - hashref to the table of dependencies as read by # . # module - The short name (just the name) of the kde-project module to list # dependencies of. # branch - The branch to assume for module. This must be specified, but use # '*' if you have no specific branch in mind. # # Returns: # A list of dependencies. Every item of the list will be of the form # "$moduleName:$branch", where $moduleName will be the short kde-project module # name (e.g. kdelibs) and $branch will be a specific git branch or '*'. # The order of the entries within the list is not important. sub _directDependenciesOf { my ($dependenciesOfRef, $module, $branch) = @_; my $moduleDepEntryRef = $dependenciesOfRef->{"$module:*"}; my @directDeps; my @exclusions; return unless $moduleDepEntryRef; push @directDeps, @{$moduleDepEntryRef->{'+'}}; push @exclusions, @{$moduleDepEntryRef->{'-'}}; $moduleDepEntryRef = $dependenciesOfRef->{"$module:$branch"}; if ($moduleDepEntryRef && $branch ne '*') { push @directDeps, @{$moduleDepEntryRef->{'+'}}; push @exclusions, @{$moduleDepEntryRef->{'-'}}; } foreach my $exclusion (@exclusions) { # Remove only modules at the exact given branch as a dep. # However catch-alls can remove catch-alls. # But catch-alls cannot remove a specific branch, such exclusions have # to also be specific. @directDeps = grep { $_ ne $exclusion } (@directDeps); } return @directDeps; } +sub _lookupDirectDependencies +{ + my $self = assert_isa(shift, 'ksb::DependencyResolver'); + my ($path, $branch) = @_; + + my $dependenciesOfRef = $self->{dependenciesOf}; + + my @directDeps = (); + my @exclusions = (); + + my $item = _shortenModuleName($path); + my $moduleDepEntryRef = $dependenciesOfRef->{"$item:*"}; + + if ($moduleDepEntryRef) { + debug("handling dependencies for: $item without branch (*)"); + push @directDeps, @{$moduleDepEntryRef->{'+'}}; + push @exclusions, @{$moduleDepEntryRef->{'-'}}; + } + + if ($branch && $branch ne '*') { + $moduleDepEntryRef = $dependenciesOfRef->{"$item:$branch"}; + if ($moduleDepEntryRef) { + push @directDeps, @{$moduleDepEntryRef->{'+'}}; + push @exclusions, @{$moduleDepEntryRef->{'-'}}; + } + } + + while (my ($catchAll, $deps) = each %{$self->{catchAllDependencies}}) { + my $prefix = $catchAll; + $prefix =~ s/\*$//; + + if (($path =~ /^$prefix/) || !$prefix) { + push @directDeps, @{$deps}; + } + } + + foreach my $exclusion (@exclusions) { + # Remove only modules at the exact given branch as a dep. + # However catch-alls can remove catch-alls. + # But catch-alls cannot remove a specific branch, such exclusions have + # to also be specific. + @directDeps = grep { $_ ne $exclusion } (@directDeps); + } + + @directDeps = uniq(@directDeps); + + return @directDeps; +} + +sub _toDependencyItemNames +{ + my @deps = @_; + my @names = map { + my $dep = $_; + my ($depItem, $depBranch) = ($dep =~ m/^([^:]+):(.*)$/); + error("Invalid dependency item: b[$dep]") if !$depItem; + $depItem; + } @deps; + + my @valid = grep { $_ } (@names); + + return uniq(@valid); +} + +sub _voteForDependencies +{ + my ($moduleGraph, $item, $from) = @_; + + if ($from ne $item) { + ++($moduleGraph->{$item}->{votes}->{$from}); + } + + my @names = @{$moduleGraph->{$item}->{itemNames}}; + for my $name (@names) { + _voteForDependencies($moduleGraph, $name, $from); + } +} + +sub _runDependencyVote +{ + my $moduleGraph = shift; + + for my $item (keys(%$moduleGraph)) { + _voteForDependencies($moduleGraph, $item, $item); + } + + return $moduleGraph; +} + +our $WELL_KNOWN_DEPENDENCY_CYCLES = { + 'Qt5' => { + 'Qt5' => 1, + 'extra-cmake-modules' => 1 + }, + 'extra-cmake-modules' => { + 'extra-cmake-modules' => 1 + } +}; + +sub _breakWellKnownDependencyCycles +{ + my $moduleGraph = shift; + + for my $item (keys(%$moduleGraph)) { + my $cyclesToBreak = $WELL_KNOWN_DEPENDENCY_CYCLES->{$item}; + next if (!$cyclesToBreak); + + debug("Breaking well known dependency cycles at: b[$item]"); + my @orig = @{$moduleGraph->{$item}->{itemNames}}; + my @broken = grep { !$cyclesToBreak->{$_} } (@orig); + my $removed = scalar(@orig) - scalar(@broken); + + if($removed) { + $moduleGraph->{$item}->{itemNames} = []; + my $ref = $moduleGraph->{$item}->{itemNames}; + push @$ref, @broken; + debug("\tNumber of cycles broken at b[$item]: $removed"); + } else { + warning("\tNo cycles broken at b[$item] please file a bug against kdesrc-build about this!"); + } + } + + return $moduleGraph; +} + +sub _detectDependencyCycle +{ + my ($moduleGraph, $depItem, $item) = @_; + + my $depModuleGraph = $moduleGraph->{$depItem}; + if ($depModuleGraph->{traces}->{status}) { + if ($depModuleGraph->{traces}->{status} == 2) { + debug("Already resolved $depItem -- skipping"); + return $depModuleGraph->{traces}->{result}; + } + else { + error("Found a dependency cycle at: $depItem while tracing $item"); + $depModuleGraph->{traces}->{result} = 1; + } + } + else { + $depModuleGraph->{traces}->{status} = 1; + $depModuleGraph->{traces}->{result} = 0; + + my @names = @{$depModuleGraph->{itemNames}}; + + for my $name (@names) { + if (_detectDependencyCycle($moduleGraph, $name, $item)) { + $depModuleGraph->{traces}->{result} = 1; + } + } + } + + $depModuleGraph->{traces}->{status} = 2; + return $depModuleGraph->{traces}->{result}; +} + +sub _checkDependencyCycles +{ + my $moduleGraph = shift; + + my $errors = 0; + + for my $item (keys(%$moduleGraph)) { + if(_detectDependencyCycle($moduleGraph, $item, $item)) { + error("Somehow there is a circular dependency involving b[$item]! :("); + error("Please file a bug against kde-build-metadata about this!"); + ++$errors; + } + } + + if($errors) { + error("Total of items with at least one circular dependency detected: $errors"); + } + + return $errors ? undef: $moduleGraph; +} + +sub _detectBranchConflict +{ + my ($moduleGraph, $item, $branch) = @_; + + if ($branch) { + my $subGraph = $moduleGraph->{$item}; + my $previouslySelectedBranch = $subGraph->{branch}; + + return $previouslySelectedBranch if($previouslySelectedBranch && $previouslySelectedBranch ne $branch); + } + + return undef; +} + +our $WELL_KNOWN_MODULE_PATHS_BY_TYPE = { + 'Qt5' => 'Qt5' +}; + +sub _getDependencyPathOf +{ + my ($module, $item, $path) = @_; + my $result = undef; + + if ($module) { + if ($module->isKDEProject()) { + my $kdeProjectPath = $module->fullProjectPath(); + debug("\tUsing full project path: 'b[$kdeProjectPath]' for item: b[$item]"); + return $kdeProjectPath; + } + + # + # A given module may not be a KDE module. + # In that case ... a work-around is needed. + # Fortunately, there seems to be just one case: Qt5 + # + my $type = $module->buildSystemType(); + my $wellKnownPathByType = $WELL_KNOWN_MODULE_PATHS_BY_TYPE->{$type}; + if ($wellKnownPathByType) { + debug("\tUsing well-known module path: 'b[$wellKnownPathByType]' for item: b[$item]"); + return $wellKnownPathByType; + } + } + + debug("\tGuessing path: 'b[$path]' for item: b[$item]"); + return $path; +} + +sub _resolveDependenciesForModuleDescription +{ + my $self = assert_isa(shift, 'ksb::DependencyResolver'); + my ($moduleGraph, $moduleDesc) = @_; + + my $module = $moduleDesc->{module}; + if($module) { + assert_isa($module, 'ksb::Module'); + } + + my $path = $moduleDesc->{path}; + my $item = $moduleDesc->{item}; + my $branch = $moduleDesc->{branch}; + my $includeDependencies = $module + ? $module->getOption('include-dependencies') + : $moduleDesc->{includeDependencies}; + my $branchErrors = 0; + + my $subGraph = $moduleGraph->{$item}; + + if(!$subGraph) { + croak_internal("Should not get here: detected a gap in resolved dependency graph at: b[$item] :("); + } + + debug("Resolving dependencies for module: $item"); + + my @dependencies = @{$subGraph->{deps}}; + + for my $dep (@dependencies) { + # maybe $moduleName:branch, maybe $moduleName:* -> split into item, branch + # lookup module if found -> let it be $depModule + # check if $moduleGraph has a the vote for it + + my ($depPath, $depBranch) = ($dep =~ m/^([^:]+):(.*)$/); + $depPath //= ''; + $depBranch //= ''; + croak_internal("Invalid dependency item: b[$dep]") if !$depPath; + + my $depItem = _shortenModuleName($depPath); + + debug ("\tdep-resolv: b[$item:", $branch ? "$branch" : "*", "] depends on b[$depItem:", $depBranch ? "$depBranch" : "*", "]"); + + my $depModuleGraph = $moduleGraph->{$depItem}; + + # work-around: wildcard branches are a don't care, not an actual branch name/value + $depBranch = undef if ($depBranch eq '*'); + + if($depModuleGraph) { + my $previouslySelectedBranch = _detectBranchConflict($moduleGraph, $depItem, $depBranch); + if($previouslySelectedBranch) { + error("r[Found a dependency conflict in branches ('b[$previouslySelectedBranch]' is not 'b[$depBranch]') for b[$depItem]! :("); + ++$branchErrors; + } + else { + if($depBranch) { + $depModuleGraph->{branch} = $depBranch; + } + } + } + else { + my $depModule = $self->{moduleFactoryRef}($depItem); + my $resolvedPath = _getDependencyPathOf($depModule, $depItem, $depPath); + # May not exist, e.g. misspellings or 'virtual' dependencies like kf5umbrella. + if(!$depModule) { + debug("\tdep-resolve: Will not build virtual or undefined module: b[$depItem]\n"); + } + + $moduleGraph->{$depItem} = { + votes => {}, + path => $resolvedPath, + build => $depModule && $includeDependencies ? 1 : 0, + branch => $depBranch, + deps => [], + module => $depModule, + itemNames => [], + traces => {} + }; + my $depModuleDesc = { + includeDependencies => $includeDependencies, + module => $depModule, + item => $depItem, + path => $resolvedPath, + branch => $depBranch + }; + + my $depsRef = $moduleGraph->{$depItem}->{deps}; + my @deps = $self->_lookupDirectDependencies($depModuleDesc->{path}, $depModuleDesc->{branch}); + push @$depsRef, @deps; + + my $namesRef = $moduleGraph->{$depItem}->{itemNames}; + my @names = _toDependencyItemNames(@deps); + push @$namesRef, @names; + + if (!$moduleGraph->{$depItem}->{build}) { + debug (" y[b[*] $item depends on $depItem, but no module builds $depItem for this run.]"); + } + + if($depModule && $depBranch && (_getBranchOf($depModule) // '') ne "$depBranch") { + my $wrongBranch = _getBranchOf($depModule) // '?'; + error(" r[b[*] $item needs $depItem:$depBranch, not $depItem:$wrongBranch]"); + ++$branchErrors; + } + + debug("Resolving transitive dependencies for module: b[$item] (via: b[$dep])"); + $branchErrors += $self->_resolveDependenciesForModuleDescription($moduleGraph, $depModuleDesc); + } + } + + return $branchErrors; +} + +sub resolveToModuleGraph +{ + my $self = assert_isa(shift, 'ksb::DependencyResolver'); + my @modules = @_; + + my %graph; + my $moduleGraph = \%graph; + + my $branchErrors = 0; + my $pathErrors = 0; + + for my $module (@modules) { + my $item = $module->name(); # _shortenModuleName($path); + my $branch = _getBranchOf($module); + my $path = _getDependencyPathOf($module, $item, ''); + + if (!$path) { + error("r[Unable to determine project/dependency path of module: $item]"); + ++$pathErrors; + next; + } + + if($moduleGraph->{$item}) { + debug("Module pulled in previously through (transitive) dependencies: $item"); + my $previouslySelectedBranch = _detectBranchConflict($moduleGraph, $item, $branch); + if($previouslySelectedBranch) { + error("r[Found a dependency conflict in branches ('b[$previouslySelectedBranch]' is not 'b[$branch]') for b[$item]! :("); + ++$branchErrors; + } + else { + if($branch) { + $moduleGraph->{$item}->{branch} = $branch; + } + } + } + else { + $moduleGraph->{$item} = { + votes => {}, + path => $path, + build => 1, + branch => $branch, + module => $module, + deps => [], + itemNames => [], + traces => {} + }; + + my $depsRef = $moduleGraph->{$item}->{deps}; + my @deps = $self->_lookupDirectDependencies($path, $branch); + push @$depsRef, @deps; + + my $namesRef = $moduleGraph->{$item}->{itemNames}; + my @names = _toDependencyItemNames(@deps); + push @$namesRef, @names; + + my $moduleDesc = { + includeDependencies => $module->getOption('include-dependencies'), + path => $path, + item => $item, + branch => $branch, + module => $module + }; + $branchErrors += $self->_resolveDependenciesForModuleDescription($moduleGraph, $moduleDesc); + } + } + + if ($pathErrors) { + error("Unable to resolve requested dependency graph"); + return undef; + } + + if ($branchErrors) { + error("Total of branch conflicts detected: $branchErrors"); + return undef; + } + + $moduleGraph = _breakWellKnownDependencyCycles($moduleGraph); + $moduleGraph = _checkDependencyCycles($moduleGraph); + + if ($moduleGraph) { + return _runDependencyVote($moduleGraph); + } + else { + error("r[One or more dependency cycles were detected]"); + return undef; + } +} + +sub _descendModuleGraph +{ + my ($moduleGraph, $callback, $nodeInfo, $context) = @_; + + my $depth = $nodeInfo->{depth}; + my $index = $nodeInfo->{idx}; + my $count = $nodeInfo->{count}; + my $currentItem = $nodeInfo->{currentItem}; + my $currentBranch = $nodeInfo->{currentBranch}; + my $parentItem = $nodeInfo->{parentItem}; + my $parentBranch = $nodeInfo->{parentBranch}; + + my $subGraph = $moduleGraph->{$currentItem}; + &$callback($nodeInfo, $subGraph->{module}, $context); + + ++$depth; + + my @items = @{$subGraph->{itemNames}}; + + my $itemCount = scalar(@items); + my $itemIndex = 1; + + for my $item (@items) + { + $subGraph = $moduleGraph->{$item}; + my $branch = $subGraph->{branch} // ''; + my $itemInfo = { + build => $subGraph->{build}, + depth => $depth, + idx => $itemIndex, + count => $itemCount, + currentItem => $item, + currentBranch => $branch, + parentItem => $currentItem, + parentBranch => $currentBranch + }; + _descendModuleGraph($moduleGraph, $callback, $itemInfo, $context); + ++$itemIndex; + } +} + +sub walkModuleDependencyTrees +{ + my $moduleGraph = shift; + my $callback = shift; + my $context = shift; + my @modules = @_; + my $itemCount = scalar(@modules); + my $itemIndex = 1; + + for my $module (@modules) { + assert_isa($module, 'ksb::Module'); + my $item = $module->name(); + my $subGraph = $moduleGraph->{$item}; + my $branch = $subGraph->{branch} // ''; + my $info = { + build => $subGraph->{build}, + depth => 0, + idx => $itemIndex, + count => $itemCount, + currentItem => $item, + currentBranch => $branch, + parentItem => '', + parentBranch => '' + }; + _descendModuleGraph($moduleGraph, $callback, $info, $context); + ++$itemIndex; + } +} + +sub sortModulesIntoBuildOrder +{ + my $moduleGraph = shift; + + my @resolved = keys(%{$moduleGraph}); + my @built = grep { $moduleGraph->{$_}->{build} && $moduleGraph->{$_}->{module} } (@resolved); + my @prioritised = sort { + my $bDependsOnA = $moduleGraph->{$a}->{votes}->{$b} // 0; + my $aDependsOnB = $moduleGraph->{$b}->{votes}->{$a} // 0; + my $order = $aDependsOnB <=> $bDependsOnA; + + $order == 0 ? (scalar($moduleGraph->{$b}->{votes}) <=> scalar($moduleGraph->{$a}->{votes})) : $order; + } (@built); + + my @modules = map { $moduleGraph->{$_}->{module} } (@prioritised); + + return @modules; +} + # Function: makeCatchAllRules # # Internal: # # Given the internal dependency options data and a kde-project full path, # extracts all "catch-all" rules that apply to the given item and converts # them to standard dependencies for that item. The dependency options are # then appropriately updated. # # No checks are done for logical errors (e.g. having the item depend on # itself) and no provision is made to avoid updating a module that has # already had its catch-all rules generated. # # Parameters: # optionsRef - The hashref as provided to <_visitModuleAndDependencies> # fullName - The kde-project full project path to generate dependencies for. sub _makeCatchAllRules { my ($optionsRef, $fullName) = @_; my $dependenciesOfRef = $optionsRef->{dependenciesOf}; my $item = _shortenModuleName($fullName); while (my ($catchAll, $deps) = each %{$optionsRef->{catchAllDependencies}}) { my $prefix = $catchAll; $prefix =~ s/\*$//; if (($fullName =~ /^$prefix/) || !$prefix) { my $depEntry = "$item:*"; $dependenciesOfRef->{$depEntry} //= { '-' => [ ], '+' => [ ], }; push @{$dependenciesOfRef->{$depEntry}->{'+'}}, @{$deps}; } } } # Function: getBranchOf # # Internal: # # This function extracts the branch of the given Module by calling its # scm object's branch-determining method. It also ensures that the branch # returned was really intended to be a branch (as opposed to a detached HEAD); # undef is returned when the desired commit is not a branch name, otherwise # the user-requested branch name is returned. sub _getBranchOf { my $module = shift; my ($branch, $type) = $module->scm()->_determinePreferredCheckoutSource($module); return ($type eq 'branch' ? $branch : undef); } # Function: visitModuleAndDependencies # # Internal: # # This method is used to topographically sort dependency data. It accepts a # , ensures that any KDE Projects it depends on (which are present # on the build list) are re-ordered before the module, and then adds the # to the build list (whether it is a KDE Project or not, to # preserve ordering). # # See also _visitDependencyItemAndDependencies, which actually does most of # the work of handling dependencies, and calls back to this function when it # finds Modules in the build list. # # Parameters: # optionsRef - hashref to the module dependencies, catch-all dependencies, # module build list, module name to mapping, and auxiliary data # to see if a module has already been visited. # module - The to properly order in the build list. # level - The level of recursion of this call. # dependent - Identical to the same param as _visitDependencyItemAndDependencies # # Returns: # Nothing. The proper build order can be read out from the optionsRef passed # in. sub _visitModuleAndDependencies { my ($optionsRef, $module, $level, $dependentName) = @_; assert_isa($module, 'ksb::Module'); if ($module->scmType() eq 'proj') { my $fullName = $module->fullProjectPath(); my $item = _shortenModuleName($fullName); my $branch = _getBranchOf($module) // '*'; # Since the initial build list is visited start to finish it is # possible for this module to already be in the ordered list if # reordering has already happened or if dependencies are included (i.e. # this was a dependency of some other module). return if ($optionsRef->{visitedItems}->{$item} // 0) == 3; $dependentName //= $item if $module->getOption('include-dependencies'); _visitDependencyItemAndDependencies($optionsRef, $fullName, $branch, $level, $dependentName); $optionsRef->{visitedItems}->{$item} = 3; # Mark as also in build list } $module->setOption('#dependency-level', $level); push @{$optionsRef->{properBuildOrder}}, $module; --($optionsRef->{modulesNeeded}); return; } # Function: visitDependencyItemAndDependencies # # Internal: # # This method is used by _visitModuleAndDependencies to account for dependencies # by kde-project modules across dependency items that are not actually present # in the build. # # For instance, if kde/foo/a depends on kde/lib/bar, and kde/lib/bar depends on # kde/foo/baz, then /a also depends on /baz and should be ordered after /baz. # This function accounts for that in cases such as trying to build only /a and # /baz. # # Parameters: # optionsRef - hashref to the module dependencies, catch-all dependencies, # module build list, module name to mapping, and auxiliary data # to see if a module has already been visited. # dependencyFullItem - a string containing the full kde-projects path for the # the module. The full path is needed to handle wildcarded dependencies. # branch - The specific branch name for the dependency if # needed. The branch name is '*' if the branch doesn't matter (or can be # determined only by the branch-group in use). E.g. '*' or 'master'. # level - Level of recursion of the current call. # dependent - *if set*, is the name of the module that requires that all of its # dependencies be added to the build list (properly ordered) even if not # specifically selected in the configuration file or command line. If not set, # recursive dependencies are not pulled into the build even if they are not # in the build list. # # Returns: # Nothing. The proper build order can be read out from the optionsRef passed # in. Note that the generated build list might be longer than the build list that # was input, in the case that recursive dependency inclusion was requested. sub _visitDependencyItemAndDependencies { my ($optionsRef, $dependencyFullItem, $branch, $level, $dependentName) = @_; my $visitedItemsRef = $optionsRef->{visitedItems}; my $properBuildOrderRef = $optionsRef->{properBuildOrder}; my $dependenciesOfRef = $optionsRef->{dependenciesOf}; my $modulesFromNameRef = $optionsRef->{modulesFromName}; my $moduleFactoryRef = $optionsRef->{moduleFactoryRef}; $level //= 0; my $item = _shortenModuleName($dependencyFullItem); debug ("dep-resolv: Visiting ", (' ' x $level), "$item"); $visitedItemsRef->{$item} //= 0; # This module may have already been added to build. # 0 == Not visited # 1 == Currently visiting. Running into a module in visit state 1 indicates a cycle. # 2 == Visited, but not in build (this may happen for common dependencies with siblings, or for # modules that are not in our build list but are part of dependency chain for other modules # that *are* in build list). # 3 == Visited, placed in build queue. return if $visitedItemsRef->{$item} >= 2; # But if the value is 2 that means we've detected a cycle. if ($visitedItemsRef->{$item} == 1) { croak_internal("Somehow there is a dependency cycle involving $item! :("); } $visitedItemsRef->{$item} = 1; # Mark as currently-visiting for cycle detection. _makeCatchAllRules($optionsRef, $dependencyFullItem); for my $subItem (_directDependenciesOf($dependenciesOfRef, $item, $branch)) { my ($subItemName, $subItemBranch) = ($subItem =~ m/^([^:]+):(.*)$/); croak_internal("Invalid dependency item: $subItem") if !$subItemName; next if $subItemName eq $item; # Catch-all deps might make this happen # This keeps us from doing a deep recursive search for dependencies # on an item we've already asked about. next if (($visitedItemsRef->{$subItemName} // 0) >= 2); debug ("\tdep-resolv: $item:$branch depends on $subItem"); my $subModule = $modulesFromNameRef->{$subItemName}; if (!$subModule && $dependentName) { # Dependent item not in the build, but we're including dependencies $subModule = $moduleFactoryRef->($subItemName); # May not exist, e.g. misspellings or 'virtual' dependencies like # kf5umbrella. But if it does, update the admin for our visit. if ($subModule) { $modulesFromNameRef->{$subModule->name()} = $subModule; ++($optionsRef->{modulesNeeded}); } } if (!$subModule) { debug (" y[b[*] $item depends on $subItem, but no module builds $subItem for this run."); _visitDependencyItemAndDependencies($optionsRef, $subItemName, $subItemBranch, $level + 1, $dependentName); } else { if ($subItemBranch ne '*' && (_getBranchOf($subModule) // '') ne $subItemBranch) { my $wrongBranch = _getBranchOf($subModule) // '?'; error (" r[b[*] $item needs $subItem, not $subItemName:$wrongBranch"); } _visitModuleAndDependencies($optionsRef, $subModule, $level + 1, $dependentName); } last if $optionsRef->{modulesNeeded} == 0; } # Mark as done visiting. $visitedItemsRef->{$item} = 2; return; } # Function: resolveDependencies # # This method takes a list of Modules (real objects, not just # module names). # # These modules have their dependencies resolved, and a new list of # is returned, containing the proper build order for the module given. # # Only "KDE Project" modules can be re-ordered or otherwise affect the # build so this currently won't affect Subversion modules or "plain Git" # modules. # # The dependency data must have been read in first (). # # Parameters: # # $self - The DependencyResolver object. # @modules - List of to evaluate, in suggested build order. # # Returns: # # Modules to build, with the included KDE Project modules in a valid ordering # based on the currently-read dependency data. KDE Project modules are only # re-ordered amongst themselves, other module types retain their relative # positions. sub resolveDependencies { my $self = assert_isa(shift, 'ksb::DependencyResolver'); my @modules = @_; my $optionsRef = { visitedItems => { }, properBuildOrder => [ ], dependenciesOf => $self->{dependenciesOf}, catchAllDependencies => $self->{catchAllDependencies}, # will map names back to their Modules modulesFromName => { map { $_->name() => $_ } grep { $_->scmType() eq 'proj' } @modules }, moduleFactoryRef => $self->{moduleFactoryRef}, # Help _visitModuleAndDependencies to optimize modulesNeeded => scalar @modules, }; for my $module (@modules) { _visitModuleAndDependencies($optionsRef, $module); } return @{$optionsRef->{properBuildOrder}}; } 1;