diff --git a/modules/ksb/ModuleResolver.pm b/modules/ksb/ModuleResolver.pm index 0f18a41..b919b7e 100644 --- a/modules/ksb/ModuleResolver.pm +++ b/modules/ksb/ModuleResolver.pm @@ -1,612 +1,616 @@ package ksb::ModuleResolver 0.20; # Handle proper resolution of module selectors, including option # handling. See POD docs below for more details. use warnings; use 5.014; use ksb::Debug; use ksb::Util; use ksb::ModuleSet::KDEProjects; use ksb::Module; use List::Util qw(first); # Public API sub new { my ($class, $ctx) = @_; my $self = { context => $ctx, ignoredSelectors => [ ], # Read in from rc-file inputModulesAndOptions => [ ], cmdlineOptions => { }, deferredOptions => { }, # Holds Modules defined in course of expanding module-sets definedModules => { }, # Holds use-module mentions with their source module-set referencedModules => { }, }; return bless $self, $class; } sub setCmdlineOptions { my ($self, $cmdlineOptionsRef) = @_; $self->{cmdlineOptions} = $cmdlineOptionsRef; return; } sub setDeferredOptions { my ($self, $deferredOptionsRef) = @_; $self->{deferredOptions} = $deferredOptionsRef; return; } sub setIgnoredSelectors { my ($self, $ignoredSelectorsRef) = @_; $self->{ignoredSelectors} = $ignoredSelectorsRef // [ ]; return; } sub setInputModulesAndOptions { my ($self, $modOptsRef) = @_; $self->{inputModulesAndOptions} = $modOptsRef; # Build lookup tables $self->{definedModules} = { map { $_->name() => $_ } (@$modOptsRef) }; $self->{referencedModules} = { _listReferencedModules(@{$modOptsRef}) }; return; } # Applies cmdline and deferred options to the given modules or module-sets. sub _applyOptions { my ($self, @modules) = @_; my $cmdlineOptionsRef = $self->{cmdlineOptions}; my $deferredOptionsRef = $self->{deferredOptions}; foreach my $m (@modules) { my $name = $m->name(); # Apply deferred options first $m->setOption(%{$deferredOptionsRef->{$name} // {}}); $m->getLogDir() if $m->isa('ksb::Module'); # Most of time cmdline options will be empty if (%$cmdlineOptionsRef) { my %moduleCmdlineArgs = ( # order is important here %{$cmdlineOptionsRef->{global} // {}}, %{$cmdlineOptionsRef->{$name} // {}}, ); # Remove any options that would interfere with cmdline args # to avoid any override behaviors in setOption() delete @{$m->{options}}{keys %moduleCmdlineArgs}; # Reapply module-specific cmdline options $m->setOption(%moduleCmdlineArgs); } } return; } # Returns a hash table of all module names referenced in use-module # declarations for any ModuleSets included within the input list. Each entry # in the hash table will map the referenced module name to the source # ModuleSet. sub _listReferencedModules { my %setEntryLookupTable; my @results; for my $moduleSet (grep { $_->isa('ksb::ModuleSet') } (@_)) { @results = $moduleSet->moduleNamesToFind(); # The parens in front of 'x' are semantically required for repetition! @setEntryLookupTable{@results} = ($moduleSet) x scalar @results; } return %setEntryLookupTable; } # Expands out a single module-set listed in referencedModules and places any # ksb::Modules created as a result within the lookup table of Modules. # Returns the list of created ksb::Modules sub _expandSingleModuleSet { my $self = shift; my $neededModuleSet = shift; my $selectedReason = 'partial-expansion:' . $neededModuleSet->name(); my $lookupTableRef = $self->{definedModules}; my $setEntryLookupTableRef = $self->{referencedModules}; # expandModuleSets applies pending/cmdline options already. my @moduleResults = $self->expandModuleSets($neededModuleSet); if (!@moduleResults) { croak_runtime ("$neededModuleSet->name() expanded to an empty list of modules!"); } $_->setOption('#selected-by', $selectedReason) foreach @moduleResults; # Copy entries into the lookup table, especially in case they're # from case 3 @{$lookupTableRef}{map { $_->name() } @moduleResults} = @moduleResults; # Ensure Case 2 and Case 1 stays disjoint (our selectors should now be # in the lookup table if it uniquely matches a module at all). my @moduleSetReferents = grep { $setEntryLookupTableRef->{$_} == $neededModuleSet } (keys %$setEntryLookupTableRef); delete @{$setEntryLookupTableRef}{@moduleSetReferents}; return @moduleResults; } # Determines the most appropriate module to return for a given selector. # The selector may refer to a module or module-set, which means that the # return value may be a list of modules. sub _resolveSingleSelector { my $self = shift; my $selector = shift; my $ctx = $self->{context}; my $selectorName = $selector; my @results; # Will default to '$selector' if unset by end of sub # In the remainder of this code, lookupTableRef is basically handling # case 1, while setEntryLookupTableRef handles case 2. No ksb::Modules # are *both* case 1 and 2 at the same time, and a module-set can only # be case 1. We clean up and handle any case 3s (if any) at the end. my $lookupTableRef = $self->{definedModules}; my $setEntryLookupTableRef = $self->{referencedModules}; # Module selectors beginning with '+' force treatment as a kde-projects # module, which means they won't be matched here (we're only looking for # sets). my $forcedToKDEProject = substr($selectorName, 0, 1) eq '+'; substr($selectorName, 0, 1, '') if $forcedToKDEProject; # Checks cmdline options only my $includingDeps = exists $self->{cmdlineOptions}->{$selectorName}->{'include-dependencies'} || exists $self->{cmdlineOptions}->{'global'}->{'include-dependencies'}; # See resolveSelectorsIntoModules for what the 3 "cases" mentioned below are. # Case 2. We make these checks first since they may update %lookupTable if (exists $setEntryLookupTableRef->{$selectorName} && !exists $lookupTableRef->{$selectorName}) { my $neededModuleSet = $setEntryLookupTableRef->{$selectorName}; my @moduleResults = $self->_expandSingleModuleSet($neededModuleSet); if (!$includingDeps) { $_->setOption('include-dependencies', 0) foreach @moduleResults; } # Now lookupTable should be updated with expanded modules. $selector = $lookupTableRef->{$selectorName} // undef; # If the selector doesn't match a name exactly it probably matches # a wildcard prefix. e.g. 'kdeedu' as a selector would pull in all kdeedu/* # modules, but kdeedu is not a module-name itself anymore. In this # case just return all the modules in the expanded list. - push @results, @moduleResults unless $selector; + if (!$selector) { + push @results, map { + $_->setOption('#selected-by', 'prefix'); $_ + } (@moduleResults); + } } # Case 1 elsif (exists $lookupTableRef->{$selectorName}) { $selector = $lookupTableRef->{$selectorName}; $selector->setOption('#selected-by', 'name'); if (!$selector->isa('ksb::ModuleSet') && !$includingDeps) { # modules were manually selected on cmdline, so ignore # module-based include-dependencies, unless # include-dependencies also set on cmdline. $selector->setOption('#include-dependencies', 0); } } elsif (ref $selector && $selector->isa('ksb::Module')) { # We couldn't find anything better than what we were provided, # just use it. $selector->setOption('#selected-by', 'best-guess-after-full-search'); } elsif ($forcedToKDEProject) { # Just assume it's a kde-projects module and expand away... $selector = ksb::ModuleSet::KDEProjects->new($ctx, '_cmdline'); $selector->setModulesToFind($selectorName); $selector->setOption('#include-dependencies', $includingDeps); } else { # Case 3? $selector = ksb::Module->new($ctx, $selectorName); $selector->phases()->phases($ctx->phases()->phases()); if ($selectorName eq 'l10n') { $_->setScmType('l10n') } $selector->setScmType('proj'); $selector->setOption('#guessed-kde-project', 1); $selector->setOption('#selected-by', 'initial-guess'); $selector->setOption('#include-dependencies', $includingDeps); } push @results, $selector unless @results; return @results; } sub _expandAllUnexpandedModuleSets { my $self = shift; my @unexpandedModuleSets = unique_items(values %{$self->{referencedModules}}); $self->_expandSingleModuleSet($_) foreach @unexpandedModuleSets; return; } sub _resolveGuessedModules { my $self = shift; my $ctx = $self->{context}; my @modules = @_; # We didn't necessarily fully expand all module-sets available in the # inputModulesAndOptions when we were resolving selectors. # Because of this we may need to go a step further and expand out all # remaining module-sets in rcFileModulesAndModuleSets if we have 'guess' # modules still left over (since they might be Case 3), and see if we can # then successfully match. if (!first { $_->getOption('#guessed-kde-project', 'module') } @modules) { return @modules; } my $lookupTableRef = $self->{definedModules}; $self->_expandAllUnexpandedModuleSets(); my @results; # We use foreach since we *want* to be able to replace the iterated variable # if we find an existing module. for my $guessedModule (@modules) { if (!$guessedModule->getOption('#guessed-kde-project', 'module')) { push @results, $guessedModule; next; } # If the module we want could be found from within our rc-file # module-sets (even implicitly), use it. Otherwise assume # kde-projects and evaluate now. if (exists $lookupTableRef->{$guessedModule->name()}) { $guessedModule = $lookupTableRef->{$guessedModule->name()}; push @results, $guessedModule; } else { my $set = ksb::ModuleSet::KDEProjects->new($ctx, "guessed_from_cmdline"); $set->setModulesToFind($guessedModule->name()); my @setResults = $self->expandModuleSets($set); my $searchItem = $guessedModule->name(); if (!@setResults) { croak_runtime ("$searchItem doesn't match any modules."); } my $foundModule = first { $_->name() eq $searchItem } @setResults; $guessedModule = $foundModule if $foundModule; push @results, @setResults; } } return @results; } # Resolves already-stored module selectors into ksb::Modules, based on # the options, modules, and module-sets set. # # Returns a list of ksb::Modules in build order, with any module-sets fully # expanded. The desired options will be set for each ksb::Module returned. sub resolveSelectorsIntoModules { my ($self, @selectors) = @_; my $ctx = $self->{context}; # Basically there are 3 types of selectors at this point: # 1. Directly named and defined modules or module-sets. # 2. Referenced (but undefined) modules. These are mentioned in a # use-modules in a module set but not actually available as ksb::Module # objects yet. But we know they will exist. # 3. Indirect modules. These are modules that do exist in the KDE project # metadata, and will be pulled in once all module-sets are expanded # (whether that's due to implicit wildcarding with use-modules, or due # to dependency following). However we don't even know the names for # these yet. # We have to be careful to maintain order of selectors throughout. my @outputList; for my $selector (@selectors) { next if list_has ($self->{ignoredSelectors}, $selector); push @outputList, $self->_resolveSingleSelector($selector); } my @modules = $self->expandModuleSets(@outputList); # If we have any 'guessed' modules then they had no obvious source in the # rc-file. But they might still be implicitly from one of our module-sets # (Case 3). # We want them to use ksb::Modules from the rc-file modules/module-sets # instead of our shell Modules, if possible. @modules = $self->_resolveGuessedModules(@modules); return @modules; } # Similar to resolveSelectorsIntoModules, except that in this case no # 'guessing' for Modules is allowed; the requested module is returned if # present, or undef otherwise. Also unlike resolveSelectorsIntoModules, no # exceptions are thrown if the module is not present. # # The only major side-effect is that all known module-sets are expanded if # necessary before resorting to returning undef. sub resolveModuleIfPresent { my ($self, $moduleName) = @_; if (%{$self->{referencedModules}}) { $self->_expandAllUnexpandedModuleSets(); } # We may not already know about modules that can be found in kde-projects, # so double-check by resolving module name into a kde-projects module-set # selector (the + syntax) and then expanding out the module-set so generated. if (!defined $self->{definedModules}->{$moduleName}) { # TODO: Probably better to just read in the entire XML once and then # store the module list at this point. eval { $self->_expandSingleModuleSet( $self->_resolveSingleSelector("+$moduleName")); }; } return $self->{definedModules}->{$moduleName} // undef; } # Replaces ModuleSets in the given list with their component Modules, and # returns the new list. sub expandModuleSets { my $self = shift; my $ctx = $self->{context}; my @buildModuleList = @_; my @returnList; foreach my $set (@buildModuleList) { my @results = $set; # If a module-set, need to update first so it can then apply its # settings to modules it creates, otherwise update Module directly. $self->_applyOptions($set); if ($set->isa('ksb::ModuleSet')) { @results = $set->convertToModules($ctx); $self->_applyOptions(@results); } push @returnList, @results; } return @returnList; } # Internal API 1; __END__ =head1 ModuleResolver A class that handles general management tasks associated with the module build list, including option handling and resolution of module selectors into actual modules. =head2 METHODS =over =item new Creates a new C. You must pass the appropriate C Don't forget to call setPendingOptions(), setIgnoredSelectors() and setInputModulesAndOptions(). my $resolver = ModuleResolver->new($ctx); =item setPendingOptions Sets the options that should be applied to modules when they are created. No special handling for global options is performed here (but see ksb::OptionsBase::getOption and its friends). You should pass in a hashref, where module-names are keys to values which are themselves hashrefs of option-name => value pairs: $resolver->setPendingOptions( { mod1 => { 'cmake-options' => 'foo', ... }, mod2 => { } }) =item setIgnoredSelectors Declares all selectors that should be ignored by default in the process of expanding module sets. Any modules matching these selectors would be elided from any expanded module sets by default. You should pass a listref of selectors. =item setInputModulesAndOptions Declares the list of all modules and module-sets known to the program, along with their base options. Modules should be ksb::Module objects, module-sets should be ksb::ModuleSet objects, no other types should be present in the list. You should pass a listref of Modules or ModuleSets (as appropriate). =item resolveSelectorsIntoModules Resolves the given list of module selectors into ksb::Module objects, using the pending command-line options, ignore-selectors and available modules/module-sets. Selectors always choose an available ksb::Module or ksb::ModuleSet if present (based on the name() of each Module or ModuleSet, including any use-modules entries for ModuleSet objects). If a selector cannot be directly found then ModuleSet objects may be expanded into their constitutent Module objects and the search performed again. If a selector still cannot be found an exception is thrown. Any embedded ModuleSets are expanded to Modules in the return value. The list of selected Modules is returned, in the approximate order of the input list (selectors for module-sets are expanded in arbitrary order). If you are just looking for a Module that should already be present, see resolveModuleIfPresent(). my @modules = eval { $resolver->resolveSelectorsIntoModules('kdelibs', 'juk'); } =item resolveModuleIfPresent Similar to resolveSelectorsIntoModules(), except that no exceptions are thrown if the module doesn't exist. Only a single module name is supported. =item expandModuleSets Converts any ksb::ModuleSet objects in the given list of Modules and ModuleSets into their component ksb::Module objects (with proper options set, and ignored modules not present). These component objects are spliced into the list of module-type objects, replacing the ModuleSet they came from. The list of ksb::Module objects is then returned. The list passed in is not actually modified in this process. =back =head2 IMPLEMENTATION This module uses a multi-pass option resolving system, in accordance with the way kdesrc-build handles options. Consider a simple kdesrc-buildrc: global cmake-options -DCMAKE_BUILD_TYPE=Debug ... end global module-set ms-foo cmake-options -DCMAKE_BUILD_TYPE=RelWithDebInfo repository kde-projects use-modules kde/kdemultimedia include-dependencies true end module-set options framework1 set-env BUILD_DEBUG 1 end options module taglib repository git://... branch 1.6 end module options juk cxxflags -g3 -Og custom-build-command ninja end options In this case we'd expect that a module like taglib ends up with its C derived from the global section directly, while all modules included from module set C use the C defined in the module-set. At the same time we'd expect that juk has all the options listed in ms-foo, but also the specific C and C options shown, I the juk module had been referenced during the build. There are many ways to convince kdesrc-build to add a module into its build list: =over =item 1. Mention it directly on the command line. =item 2. Include it in the kdesrc-buildrc file, either as a new C block or in a C of a C. =item 3. For KDE modules, mention a component of its project path in a C declaration within a C-based module set. E.g. the "kde/kdemultimedia" entry above, which will pull in the juk module even though "juk" is not named directly. =item 4. For KDE modules, by being a dependency of a module included from a C where the C option is set to C. This wouldn't apply to juk, but might apply to modules such as phonon. Note that "taglib" in this example would B be a dependency of juk according to kdesrc-build (although it is in reality), since taglib is not a KDE module. =back This mission of this class is to ensure that, no matter I a module ended up being selected by the user for the build list, that the same options are registered into the module, the module uses the same build and scm types, is defaulted to the right build phases, etc. To do this, this class takes the read-in options, modules, and module sets from the rc-file, the list of "selectors" requested by the user (via cmdline), any changes to the options from the cmdline, and then takes pains to ensure that any requested modules are returned via the appropriate module-set (and if no module-set can source the module, via default options). In doing so, the class must keep track of module sets, the modules included into each module set, and modules that were mentioned somehow but not already present in the known list of modules (or module sets). Since module sets can cause modules to be defined that are not mentioned anywhere within an rc-file, it may be required to completely expand all module sets in order to verify that a referenced C is B already known. =head2 OUTPUTS From the perspective of calling code, the 'outputs' of this module are lists of C objects, in the order they were selected (or mentioned in the rc-file). See expandModuleSets() and resolveSelectorsIntoModules(). Each object so returned should already have the appropriate options included (based on the cmdlineOptions member, which should be constructed as the union of rc-file and cmdline options). Note that dependency resolution is B handled by this module, see C for that. =cut diff --git a/modules/ksb/Updater/Git.pm b/modules/ksb/Updater/Git.pm index b3f662b..794510c 100644 --- a/modules/ksb/Updater/Git.pm +++ b/modules/ksb/Updater/Git.pm @@ -1,919 +1,951 @@ package ksb::Updater::Git; # Module which is responsible for updating git-based source code modules. Can # have some features overridden by subclassing (see ksb::Updater::KDEProject # for an example). +use 5.014; + use ksb::Debug; use ksb::Util; use ksb::Updater; our $VERSION = '0.10'; our @ISA = qw(ksb::Updater); use File::Basename; # basename use File::Spec; # tmpdir use POSIX qw(strftime); use List::Util qw(first); +use IPC::Cmd qw(run_forked); use ksb::IPC::Null; use constant { DEFAULT_GIT_REMOTE => 'origin', }; # scm-specific update procedure. # May change the current directory as necessary. sub updateInternal { my $self = assert_isa(shift, 'ksb::Updater::Git'); my $ipc = shift; $self->{ipc} = $ipc // ksb::IPC::Null->new(); return $self->updateCheckout(); delete $self->{ipc}; } sub name { return 'git'; } sub currentRevisionInternal { my $self = assert_isa(shift, 'ksb::Updater::Git'); return $self->commit_id('HEAD'); } # Returns the current sha1 of the given git "commit-ish". sub commit_id { my $self = assert_isa(shift, 'ksb::Updater::Git'); my $commit = shift or croak_internal("Must specify git-commit to retrieve id for"); my $module = $self->module(); my $gitdir = $module->fullpath('source') . '/.git'; # Note that the --git-dir must come before the git command itself. my ($id, undef) = filter_program_output( undef, # No filter qw/git --git-dir/, $gitdir, 'rev-parse', $commit, ); chomp $id if $id; return $id; } +sub _verifyRefPresent +{ + my ($self, $module, $repo) = @_; + my ($commitId, $commitType) = $self->_determinePreferredCheckoutSource($module); + + return 1 if pretending(); + + my $ref = (($commitType eq 'branch') ? 'refs/heads/' + : ($commitType eq 'tag') ? 'refs/tags/' + : '') . $commitId; + + my $hashref = run_forked("git ls-remote --exit-code $repo $ref", + { timeout => 10, discard_output => 1, terminate_on_parent_sudden_death => 1}); + my $result = $hashref->{exit_code}; + + return 0 if ($result == 2); # Connection successful, but ref not found + return 1 if ($result == 0); # Ref is present + + croak_runtime("git had error exit $result when verifying $ref present in repository at $repo"); +} + # Perform a git clone to checkout the latest branch of a given git module # # First parameter is the repository (typically URL) to use. # Throws an exception if it fails. sub clone { my $self = assert_isa(shift, 'ksb::Updater::Git'); my $git_repo = shift; my $module = $self->module(); my $srcdir = $module->fullpath('source'); my @args = ('--', $git_repo, $srcdir); my $ipc = $self->{ipc} // croak_internal ('Missing IPC object'); + if (!$self->_verifyRefPresent($module, $git_repo)) { + # Ignore silently unless specifically requested. See ModuleResolver.pm + # for '#selected-by' logic, this is blank if no cmdline selectors at all + info ("\tSkipping, no matching remote branch exists"); + return if (($module->getOption('#selected-by', 'module') // 'prefix') eq 'prefix'); + croak_runtime("The desired git reference is not available for $module"); + }; + note ("Cloning g[$module]"); my $result = eval { $self->installGitSnapshot() }; if ((my $e = had_an_exception()) || !$result) { warning($e->message()) if $e; note ("\tFalling back to clone of $module"); p_chdir($module->getSourceDir()); if (0 != log_command($module, 'git-clone', ['git', 'clone', @args])) { croak_runtime("Failed to make initial clone of $module"); } } $ipc->notifyPersistentOptionChange( $module->name(), 'git-cloned-repository', $git_repo); p_chdir($srcdir); my ($commitId, $commitType) = $self->_determinePreferredCheckoutSource($module); # Switch immediately to user-requested tag or branch now. if ($commitType eq 'tag') { info ("\tSwitching to specific commit g[$commitId]"); if (0 != log_command($module, 'git-checkout-commit', ['git', 'checkout', $commitId])) { croak_runtime("Failed to checkout desired commit $commitId"); } } # If not a tag, it's a defined branch elsif ($commitId ne 'master') { info ("\tSwitching to branch g[$commitId]"); if (0 != log_command($module, 'git-checkout', ['git', 'checkout', '-b', $commitId, "origin/$commitId"])) { croak_runtime("Failed to checkout desired branch $commitId"); } } # Setup user configuration if (my $name = $module->getOption('git-user')) { my ($username, $email) = ($name =~ /^([^<]+) +<([^>]+)>$/); if (!$username || !$email) { croak_runtime("Invalid username or email for git-user option: $name". " (should be in format 'User Name '"); } whisper ("\tAdding git identity $name for new git module $module"); my $result = (safe_system(qw(git config --local user.name), $username) >> 8) == 0; $result = (safe_system(qw(git config --local user.email), $email) >> 8 == 0) || $result; if (!$result) { warning ("Unable to set user.name and user.email git config for y[b[$module]!"); } } return; } sub _isDirectoryEmpty { my $dir = shift; # Empty returns are OK -- they are automatically the 'false' equivalent for # whatever context the function is called in. opendir (my $dh, $dir) or return; if (any { $_ ne '.' && $_ ne '..' } [readdir($dh)]) { close $dh; return; } close $dh; return 1; } # Either performs the initial checkout or updates the current git checkout # for git-using modules, as appropriate. # # If errors are encountered, an exception is raised. # # Returns the number of *commits* affected. sub updateCheckout { my $self = assert_isa(shift, 'ksb::Updater::Git'); my $module = $self->module(); my $srcdir = $module->fullpath('source'); if (-d "$srcdir/.git") { # Note that this function will throw an exception on failure. return $self->updateExistingClone(); } else { # Check if an existing source directory is there somehow. if (-e "$srcdir" && !_isDirectoryEmpty($srcdir)) { if ($module->getOption('#delete-my-patches')) { warning ("\tRemoving conflicting source directory " . "as allowed by --delete-my-patches"); warning ("\tRemoving b[$srcdir]"); safe_rmtree($srcdir) or croak_internal("Unable to delete $srcdir!"); } else { error (<getOption('repository'); if (!$git_repo) { croak_internal("Unable to checkout $module, you must specify a repository to use."); } $self->clone($git_repo); return 1 if pretending(); return count_command_output('git', '--git-dir', "$srcdir/.git", 'ls-files'); } return 0; } # Selects a git remote for the user's selected repository (preferring a # defined remote if available, using 'origin' otherwise). # # Assumes the current directory is already set to the source directory. # # Throws an exception on error. # # Return value: Remote name that should be used for further updates. # # See also the 'repository' module option. sub _setupBestRemote { my $self = assert_isa(shift, 'ksb::Updater::Git'); my $module = $self->module(); my $cur_repo = $module->getOption('repository'); my $ipc = $self->{ipc} // croak_internal ('Missing IPC object'); # Search for an existing remote name first. If none, add our alias. my @remoteNames = $self->bestRemoteName($cur_repo); if (!@remoteNames) { # The desired repo doesn't have a named remote, this should be # because the user switched it in the rc-file. We control the # 'origin' remote to fix this. if ($self->hasRemote(DEFAULT_GIT_REMOTE)) { if (log_command($module, 'git-update-remote', ['git', 'remote', 'set-url', DEFAULT_GIT_REMOTE, $cur_repo]) != 0) { croak_runtime("Unable to update the fetch URL for existing remote alias for $module"); } } elsif (log_command($module, 'git-remote-setup', ['git', 'remote', 'add', DEFAULT_GIT_REMOTE, $cur_repo]) != 0) { croak_runtime("Unable to add a git remote named " . DEFAULT_GIT_REMOTE . " for $cur_repo"); } push @remoteNames, DEFAULT_GIT_REMOTE; } # Make a notice if the repository we're using has moved. my $old_repo = $module->getPersistentOption('git-cloned-repository'); if ($old_repo and ($cur_repo ne $old_repo)) { note (" y[b[*]\ty[$module]'s selected repository has changed"); note (" y[b[*]\tfrom y[$old_repo]"); note (" y[b[*]\tto b[$cur_repo]"); note (" y[b[*]\tThe git remote named b[", DEFAULT_GIT_REMOTE, "] has been updated"); # Update what we think is the current repository on-disk. $ipc->notifyPersistentOptionChange( $module->name(), 'git-cloned-repository', $cur_repo); } return $remoteNames[0]; } # Completes the steps needed to update a git checkout to be checked-out to # a given remote-tracking branch. Any existing local branch with the given # branch set as upstream will be used if one exists, otherwise one will be # created. The given branch will be rebased into the local branch. # # No checkout is done, this should be performed first. # Assumes we're already in the needed source dir. # Assumes we're in a clean working directory (use git-stash to achieve # if necessary). # # First parameter is the remote to use. # Second parameter is the branch to update to. # Returns boolean success flag. # Exception may be thrown if unable to create a local branch. sub _updateToRemoteHead { my $self = shift; my ($remoteName, $branch) = @_; my $module = $self->module(); # The 'branch' option requests a given head in the user's selected # repository. Normally the remote head is mapped to a local branch, # which can have a different name. So, first we make sure the remote # head is actually available, and if it is we compare its SHA1 with # local branches to find a matching SHA1. Any local branches that are # found must also be remote-tracking. If this is all true we just # re-use that branch, otherwise we create our own remote-tracking # branch. my $branchName = $self->getRemoteBranchName($remoteName, $branch); if (!$branchName) { my $newName = $self->makeBranchname($remoteName, $branch); whisper ("\tUpdating g[$module] with new remote-tracking branch y[$newName]"); if (0 != log_command($module, 'git-checkout-branch', ['git', 'checkout', '-b', $newName, "$remoteName/$branch"])) { croak_runtime("Unable to perform a git checkout of $remoteName/$branch to a local branch of $newName"); } } else { whisper ("\tUpdating g[$module] using existing branch g[$branchName]"); if (0 != log_command($module, 'git-checkout-update', ['git', 'checkout', $branchName])) { croak_runtime("Unable to perform a git checkout to existing branch $branchName"); } # On the right branch, merge in changes. return 0 == log_command($module, 'git-rebase', ['git', 'rebase', "$remoteName/$branch"]); } return 1; } # Completes the steps needed to update a git checkout to be checked-out to # a given commit. The local checkout is left in a detached HEAD state, # even if there is a local branch which happens to be pointed to the # desired commit. Based the given commit is used directly, no rebase/merge # is performed. # # No checkout is done, this should be performed first. # Assumes we're already in the needed source dir. # Assumes we're in a clean working directory (use git-stash to achieve # if necessary). # # First parameter is the commit to update to. This can be in pretty # much any format that git itself will respect (e.g. tag, sha1, etc.). # It is recommended to use refs/$foo/$bar syntax for specificity. # Returns boolean success flag. sub _updateToDetachedHead { my ($self, $commit) = @_; my $module = $self->module(); info ("\tDetaching head to b[$commit]"); return 0 == log_command($module, 'git-checkout-commit', ['git', 'checkout', $commit]); } # Updates an already existing git checkout by running git pull. # # Throws an exception on error. # # Return parameter is the number of affected *commits*. sub updateExistingClone { my $self = assert_isa(shift, 'ksb::Updater::Git'); my $module = $self->module(); my $cur_repo = $module->getOption('repository'); my $result; p_chdir($module->fullpath('source')); # Try to save the user if they are doing a merge or rebase if (-e '.git/MERGE_HEAD' || -e '.git/rebase-merge' || -e '.git/rebase-apply') { croak_runtime ("Aborting git update for $module, you appear to have a rebase or merge in progress!"); } my $remoteName = $self->_setupBestRemote(); # Download updated objects. This also updates remote heads so do this # before we start comparing branches and such. if (0 != log_command($module, 'git-fetch', ['git', 'fetch', '--tags', $remoteName])) { croak_runtime ("Unable to perform git fetch for $remoteName ($cur_repo)"); } # Now we need to figure out if we should update a branch, or simply # checkout a specific tag/SHA1/etc. my ($commitId, $commitType) = $self->_determinePreferredCheckoutSource($module); note ("Updating g[$module] (to $commitType b[$commitId])"); my $start_commit = $self->commit_id('HEAD'); my $updateSub; if ($commitType eq 'branch') { $updateSub = sub { $self->_updateToRemoteHead($remoteName, $commitId) }; } else { $updateSub = sub { $self->_updateToDetachedHead($commitId); } } # With all remote branches fetched, and the checkout of our desired # branch completed, we can now use our update sub to complete the # changes. $self->stashAndUpdate($updateSub); return count_command_output('git', 'rev-list', "$start_commit..HEAD"); } # Goes through all the various combination of git checkout selection options in # various orders of priority. # # Returns a *list* containing: (the resultant symbolic ref/or SHA1,'branch' or # 'tag' (to determine if something like git-pull would be suitable or whether # you have a detached HEAD)). Since the sym-ref is returned first that should # be what you get in a scalar context, if that's all you want. sub _determinePreferredCheckoutSource { my ($self, $module) = @_; $module //= $self->module(); my @priorityOrderedSources = ( # option-name type getOption-inheritance-flag [qw(commit tag module)], [qw(revision tag module)], [qw(tag tag module)], [qw(branch branch module)], [qw(branch-group branch module)], [qw(use-stable-kde branch module)], # commit/rev/tag don't make sense for git as globals [qw(branch branch allow-inherit)], [qw(branch-group branch allow-inherit)], [qw(use-stable-kde branch allow-inherit)], ); # For modules that are not actually a 'proj' module we skip branch-group # and use-stable-kde entirely to allow for global/module branch selection # options to be selected... kind of complicated, but more DWIMy if (!$module->scm()->isa('ksb::Updater::KDEProject')) { @priorityOrderedSources = grep { $_->[0] ne 'branch-group' && $_->[0] ne 'use-stable-kde' } @priorityOrderedSources; } my $checkoutSource; # Sorry about the !!, easiest way to be clear that bool context is intended my $sourceTypeRef = first { !!($checkoutSource = ($module->getOption($_->[0], $_->[2]) // '')) } @priorityOrderedSources; if (!$sourceTypeRef) { return qw(master branch); } # One fixup is needed for use-stable-kde, to pull the actual branch name # from the right spot. Although if no branch name is set we use master, # without trying to search again. if ($sourceTypeRef->[0] eq 'use-stable-kde') { $checkoutSource = $module->getOption('#branch:stable', 'module') || 'master'; } # Likewise branch-group requires special handling. checkoutSource is # currently the branch-group to be resolved. if ($sourceTypeRef->[0] eq 'branch-group') { assert_isa($self, 'ksb::Updater::KDEProject'); $checkoutSource = $self->_resolveBranchGroup($checkoutSource); if (!$checkoutSource) { my $branchGroup = $module->getOption('branch-group'); whisper ("No specific branch set for $module and $branchGroup, using master!"); $checkoutSource = 'master'; } } if ($sourceTypeRef->[0] eq 'tag' && $checkoutSource !~ m{^refs/tags/}) { $checkoutSource = "refs/tags/$checkoutSource"; } return ($checkoutSource, $sourceTypeRef->[1]); } # Splits a URI up into its component parts. Taken from # http://search.cpan.org/~ether/URI-1.67/lib/URI.pm # Copyright Gisle Aas under the following terms: # "This program is free software; you can redistribute it and/or modify it # under the same terms as Perl itself." sub _splitUri { my($scheme, $authority, $path, $query, $fragment) = $_[0] =~ m|(?:([^:/?#]+):)?(?://([^/?#]*))?([^?#]*)(?:\?([^#]*))?(?:#(.*))?|; return ($scheme, $authority, $path, $query, $fragment); } # Attempts to download and install a git snapshot for the given ksb::Module. # This requires the module to have the '#snapshot-tarball' option set, # normally done after KDEXMLReader is used to parse the projects.kde.org # XML database. This function should be called with the current directory # set to the source directory. # # After installing the tarball, an immediate git pull will be run to put the # module up-to-date. The branch is not updated however! # # The user can cause this function to fail by setting the disable-snapshots # option for the module (either at the command line or in the rc file). # # Throws an exception on failure. # # Returns true if the snapshot actually was installed and usable, false # otherwise. If the snapshot was not installed, the source directory should # still be empty for it. sub installGitSnapshot { my $self = assert_isa(shift, 'ksb::Updater::Git'); my $module = $self->module(); my $baseDir = $module->getSourceDir(); my $tarball = $module->getOption('#snapshot-tarball'); return 0 if $module->getOption('disable-snapshots'); return 0 unless $tarball; if (pretending()) { pretend ("\tWould have downloaded snapshot for g[$module], from"); pretend ("\tb[g[$tarball]"); return 1; } info ("\tDownloading git snapshot for g[$module]"); my $filename = basename( (_splitUri($tarball))[2] ); my $tmpdir = File::Spec->tmpdir() // "/tmp"; $filename = "$tmpdir/$filename"; # Make absolute download_file($tarball, $filename, $module->getOption('http-proxy')) or croak_runtime("Unable to download snapshot for module \"$module\""); info ("\tDownload complete, preparing module source code"); # It would be possible to use Archive::Tar, but it's apparently fairly # slow. In addition we need to use -C and --strip-components (which are # also supported in BSD tar, perhaps not Solaris) to ensure it's extracted # in a known location. Since we're using "sufficiently good" tar programs # we can take advantage of their auto-decompression. my $sourceDir = $module->fullpath('source'); super_mkdir($sourceDir); my $result = safe_system(qw(tar --strip-components 1 -C), $sourceDir, '-xf', $filename); my $savedError = $!; # Avoid interference from safe_unlink safe_unlink ($filename); if ($result) { safe_rmtree($sourceDir); croak_runtime("Unable to extract snapshot for $module: $savedError"); } whisper ("\tg[$module] snapshot is in place"); # Complete the preparation by running the initrepo.sh script p_chdir($sourceDir); $result = log_command($module, 'init-git-repo', ['/bin/sh', './initrepo.sh']); if ($result) { p_chdir($baseDir); safe_rmtree($sourceDir); croak_runtime("Snapshot for $module failed to complete initrepo.sh"); } whisper ("\tConverting to kde:-style URL"); $result = log_command($module, 'fixup-git-remote', ['git', 'remote', 'set-url', 'origin', "kde:$module"]); if ($result) { warning ("\tUnable to convert origin URL to kde:-style URL. Things should"); warning ("\tstill work, you may have to adjust push URL manually."); } info ("\tGit snapshot installed, now bringing up to date."); $result = log_command($module, 'init-git-pull', ['git', 'pull']); if ($result != 0) { warning("Unable to complete update for y[b[$module] snapshot"); } return 1; } # This stashes existing changes if necessary, and then runs a provided # update routine in order to advance the given module to the desired head. # Finally, if changes were stashed, they are applied and the stash stack is # popped. # # It is assumed that the required remote has been setup already, that we # are on the right branch, and that we are already in the correct # directory. # # First parameter is a reference to the subroutine to run. This subroutine # should need no parameters and return a boolean success indicator. It may # throw exceptions. # # Throws an exception on error. # # No return value. sub stashAndUpdate { my $self = assert_isa(shift, 'ksb::Updater::Git'); my $updateSub = shift; my $module = $self->module(); my $date = strftime ("%F-%R", gmtime()); # ISO Date, hh:mm time # To find out if we should stash, we just use git diff --quiet, twice to # account for the index and the working dir. # Note: Don't use safe_system, as the error code is stripped to the exit code my $status = pretending() ? 0 : system('git', 'diff', '--quiet'); if ($status == -1 || $status & 127) { croak_runtime("$module doesn't appear to be a git module."); } my $needsStash = 0; if ($status) { # There are local changes. $needsStash = 1; } else { $status = pretending() ? 0 : system('git', 'diff', '--cached', '--quiet'); if ($status == -1 || $status & 127) { croak_runtime("$module doesn't appear to be a git module."); } else { $needsStash = ($status != 0); } } if ($needsStash) { info ("\tLocal changes detected, stashing them away..."); $status = log_command($module, 'git-stash-save', [ qw(git stash save --quiet), "kdesrc-build auto-stash at $date", ]); if ($status != 0) { croak_runtime("Unable to stash local changes for $module, aborting update."); } } if (!$updateSub->()) { error ("\tUnable to update the source code for r[b[$module]"); return; } # Update is performed and successful, re-apply the stashed changes if ($needsStash) { info ("\tModule updated, reapplying your local changes."); $status = log_command($module, 'git-stash-pop', [ qw(git stash pop --index --quiet) ]); if ($status != 0) { error (<module(); my $chosenName; # Use "$branch" directly if not already used, otherwise try to prefix # with the remote name. for my $possibleBranch ($branch, "$remoteName-$branch", "ksdc-$remoteName-$branch") { my $result = system('git', 'show-ref', '--quiet', '--verify', '--', "refs/heads/$possibleBranch") >> 8; return $possibleBranch if $result == 1; } croak_runtime("Unable to find good branch name for $module branch name $branch"); } # Returns the number of lines in the output of the given command. The command # and all required arguments should be passed as a normal list, and the current # directory should already be set as appropriate. # # Return value is the number of lines of output. # Exceptions are raised if the command could not be run. sub count_command_output { # Don't call with $self->, all args are passed to filter_program_output my @args = @_; my $count = 0; filter_program_output(sub { $count++ if $_ }, @args); return $count; } # A simple wrapper that is used to split the output of 'git config --null' # correctly. All parameters are then passed to filter_program_output (so look # there for help on usage). sub slurp_git_config_output { # Don't call with $self->, all args are passed to filter_program_output local $/ = "\000"; # Split on null # This gets rid of the trailing nulls for single-line output. (chomp uses # $/ instead of hardcoding newline chomp(my @output = filter_program_output(undef, @_)); # No filter return @output; } # Returns true if the git module in the current directory has a remote of the # name given by the first parameter. sub hasRemote { my ($self, $remote) = @_; my $hasRemote = 0; eval { filter_program_output(sub { $hasRemote ||= ($_ && /^$remote/) }, 'git', 'remote'); }; return $hasRemote; } # Subroutine to add the 'kde:' alias to the user's git config if it's not # already set. # # Call this as a static class function, not as an object method # (i.e. ksb::Updater::Git::verifyGitConfig, not $foo->verifyGitConfig) # # Returns false on failure of any sort, true otherwise. sub verifyGitConfig { my $configOutput = qx'git config --global --get url.git://anongit.kde.org/.insteadOf kde:'; # 0 means no error, 1 means no such section exists -- which is OK if ((my $errNum = $? >> 8) >= 2) { my $error = "Code $errNum"; my %errors = ( 3 => 'Invalid config file (~/.gitconfig)', 4 => 'Could not write to ~/.gitconfig', 2 => 'No section was provided to git-config', 1 => 'Invalid section or key', 5 => 'Tried to set option that had no (or multiple) values', 6 => 'Invalid regexp with git-config', 128 => 'HOME environment variable is not set (?)', ); $error = $errors{$errNum} if exists $errors{$errNum}; error (" r[*] Unable to run b[git] command:\n\t$error"); return 0; } # If we make it here, I'm just going to assume git works from here on out # on this simple task. if ($configOutput !~ /^kde:\s*$/) { whisper ("\tAdding git download kde: alias"); my $result = safe_system( qw(git config --global --add url.git://anongit.kde.org/.insteadOf kde:) ) >> 8; return 0 if $result != 0; } $configOutput = qx'git config --global --get url.git@git.kde.org:.pushInsteadOf kde:'; if ($configOutput !~ /^kde:\s*$/) { whisper ("\tAdding git upload kde: alias"); my $result = safe_system( qw(git config --global --add url.git@git.kde.org:.pushInsteadOf kde:) ) >> 8; return 0 if $result != 0; } return 1; } 1;