diff --git a/modules/ksb/KDEXMLReader.pm b/modules/ksb/KDEXMLReader.pm index 9a26b94..e7493f6 100644 --- a/modules/ksb/KDEXMLReader.pm +++ b/modules/ksb/KDEXMLReader.pm @@ -1,215 +1,213 @@ package ksb::KDEXMLReader 0.40; # Class: KDEXMLReader # # Enumerates and provides basic metadata of KDE projects, based on # the YAML metadata included in sysadmin/repo-management. use strict; use warnings; use 5.014; use File::Find; # Load YAML if available without causing compile error if it isn't my @YAML_Opts = qw(Dump Load LoadFile); my $success = eval { require YAML::XS; YAML::XS->import(@YAML_Opts); 1; }; # Note that YAML::Tiny does not work since some metadata files use # features it doesn't support if (!$success) { $success = eval { require YAML; # Hope that an appropriate module is mapped to this entry point YAML->import(@YAML_Opts); 1; }; } if (!$success) { die "Unable to load YAML::Tiny or YAML::XS modules, one of which is needed to handle KDE project data."; } # Method: new # # Constructs a new KDEXMLReader. This doesn't contradict any part of the class # documentation which claims this class is a singleton however. This should be # called as a method (e.g. KDEXMLReader->new(...)). # # Parameters: # $projectMetadataModule - ksb::Module reference to the repo-metadata module. # $desiredProtocol - Normally 'git', but other protocols like 'http' can also # be preferred (e.g. for proxy compliance). sub new { my $class = shift; my $projectMetadataModule = shift; my $desiredProtocol = shift; my $self = { # Maps short names to repo info blocks repositories => { }, protocol => $desiredProtocol, }; $self = bless ($self, $class); $self->_readProjectData($projectMetadataModule); return $self; } # The 'main' method for this class. Reads in *all* KDE projects and notes # their details for later queries. # Be careful, can throw exceptions. sub _readProjectData { my ($self, $projectMetadataModule) = @_; my $srcdir = $projectMetadataModule->fullpath('source'); my $file_searched = 0; File::Find::find(sub { $self->_readYAML($_) if $_ eq 'metadata.yaml'; $file_searched = 1; }, "$srcdir/projects"); } sub _readYAML { my ($self, $filename) = @_; my $proj_data = LoadFile($filename); if (!$proj_data->{repoactive} || !$proj_data->{hasrepo} || ($proj_data->{type} ne 'project' && $proj_data->{type} ne 'module')) { return; }; my $repoName = $proj_data->{repopath}; my $repoPath = $self->{protocol} . "://anongit.kde.org/$repoName"; my $curRepository = { 'fullName' => $proj_data->{projectpath}, 'repo' => $repoPath, 'name' => $repoName, 'active' => !!$proj_data->{repoactive}, - # Hardcoded from common/urls_gitrepo.json - # 'tarball' => "https://anongit.kde.org/$repoName/$repoName-latest.tar.gz", - 'tarball' => '', - - # Branch data was entered into XML by reading git repo directly, so we must - # wait for git repo to be loaded or use dependency data/logical module groups - # 'branch' => '', - # 'branches' => [ ], - # 'branchtype' => '', # Either branch:stable or branch:trunk - }; # Repo/Active/tarball to be added by char handler. + 'found_by' => 'direct', # can be changed in getModulesForProject + }; $self->{repositories}->{$repoName} = $curRepository; } # Note on $proj: A /-separated path is fine, in which case we look # for the right-most part of the full path which matches all of searchProject. # e.g. kde/kdebase/kde-runtime would be matched by a proj of either # "kdebase/kde-runtime" or simply "kde-runtime". sub getModulesForProject { my ($self, $proj) = @_; my $repositoryRef = $self->{repositories}; my @results; my $findResults = sub { - push @results, ( + my @matchList = grep { _projectPathMatchesWildcardSearch( $repositoryRef->{$_}->{'fullName'}, $proj) - } (keys %{$repositoryRef})); + } (keys %{$repositoryRef}); + + if ($proj =~ m/\*/) { + $repositoryRef->{$_}->{found_by} = 'wildcard' foreach @matchList; + } + + push @results, @matchList; }; # Wildcard matches happen as specified if asked for. # Non-wildcard matches have an implicit "$proj/*" search as well for # compatibility with previous use-modules # Project specifiers ending in .git are forced to be non-wildcarded. if ($proj !~ /\*/ && $proj !~ /\.git$/) { # We have to do a search to account for over-specified module names # like phonon/phonon $findResults->(); # Now setup for a wildcard search to find things like kde/kdelibs/baloo # if just 'kdelibs' is asked for. $proj .= '/*'; } $proj =~ s/\.git$//; # If still no wildcard and no '/' then we can use direct lookup by module # name. if ($proj !~ /\*/ && $proj !~ /\// && exists $repositoryRef->{$proj}) { push @results, $proj; } else { $findResults->(); } return @{$repositoryRef}{@results}; } # Utility subroutine, returns true if the given kde-project full path (e.g. # kde/kdelibs/nepomuk-core) matches the given search item. # # The search item itself is based on path-components. Each path component in # the search item must be present in the equivalent path component in the # module's project path for a match. A '*' in a path component position for the # search item matches any project path component. # # Finally, the search is pinned to search for a common suffix. E.g. a search # item of 'kdelibs' would match a project path of 'kde/kdelibs' but not # 'kde/kdelibs/nepomuk-core'. However 'kdelibs/*' would match # 'kde/kdelibs/nepomuk-core'. # # First parameter is the full project path from the kde-projects database. # Second parameter is the search item. # Returns true if they match, false otherwise. sub _projectPathMatchesWildcardSearch { my ($projectPath, $searchItem) = @_; my @searchParts = split(m{/}, $searchItem); my @nameStack = split(m{/}, $projectPath); if (scalar @nameStack >= scalar @searchParts) { my $sizeDifference = scalar @nameStack - scalar @searchParts; # We might have to loop if we somehow find the wrong start point for our search. # E.g. looking for a/b/* against a/a/b/c, we'd need to start with the second a. my $i = 0; while ($i <= $sizeDifference) { # Find our common prefix, then ensure the remainder matches item-for-item. for (; $i <= $sizeDifference; $i++) { last if $nameStack[$i] eq $searchParts[0]; } return if $i > $sizeDifference; # Not enough room to find it now # At this point we have synched up nameStack to searchParts, ensure they # match item-for-item. my $found = 1; for (my $j = 0; $found && ($j < @searchParts); $j++) { return 1 if $searchParts[$j] eq '*'; # This always works $found = 0 if $searchParts[$j] ne $nameStack[$i + $j]; } return 1 if $found; # We matched every item to the substring we found. $i++; # Try again } } return; } 1; diff --git a/modules/ksb/ModuleSet/KDEProjects.pm b/modules/ksb/ModuleSet/KDEProjects.pm index 1add6c4..5e3ddc3 100644 --- a/modules/ksb/ModuleSet/KDEProjects.pm +++ b/modules/ksb/ModuleSet/KDEProjects.pm @@ -1,246 +1,244 @@ package ksb::ModuleSet::KDEProjects 0.20; # Class: ModuleSet::KDEProjects # # This represents a collective grouping of modules that share common options, # and are obtained via the kde-projects database at # https://projects.kde.org/kde_projects.xml # # See also the parent class ModuleSet, from which most functionality is derived. # # The only changes here are to allow for expanding out module specifications # (except for ignored modules), by using KDEXMLReader. # # See also: ModuleSet use strict; use warnings; use 5.014; no if $] >= 5.018, 'warnings', 'experimental::smartmatch'; our @ISA = qw(ksb::ModuleSet); use ksb::Module; use ksb::Debug; use ksb::KDEXMLReader 0.20; use ksb::BuildContext 0.20; use ksb::Util; sub new { my $self = ksb::ModuleSet::new(@_); $self->{projectsDataReader} = undef; # Will be filled in when we get fh return $self; } # Simple utility subroutine. See List::Util's perldoc sub none_true { ($_ && return 0) for @_; return 1; } sub _createMetadataModule { my ($ctx, $moduleName) = @_; my $metadataModule = ksb::Module->new($ctx, $moduleName =~ s,/,-,r); # Hardcode the results instead of expanding out the project info $metadataModule->setOption('repository', "kde:$moduleName"); $metadataModule->setOption('#xml-full-path', $moduleName); $metadataModule->setOption('#branch:stable', 'master'); $metadataModule->setScmType('metadata'); $metadataModule->setOption('disable-snapshots', 1); $metadataModule->setOption('branch', 'master'); my $moduleSet = ksb::ModuleSet::KDEProjects->new($ctx, ''); $metadataModule->setModuleSet($moduleSet); # Ensure we only ever try to update source, not build. $metadataModule->phases()->phases('update'); return $metadataModule; } # Function: getDependenciesModule # # Static. Returns a that can be used to download the # 'kde-build-metadata' module, which itself contains module dependencies # in the KDE build system. The module is meant to be held by the # # Parameters: # ctx - the for this script execution. sub getDependenciesModule { my $ctx = assert_isa(shift, 'ksb::BuildContext'); return _createMetadataModule($ctx, 'kde-build-metadata'); } # Function: getProjectMetadataModule # # Static. Returns a that can be used to download the # 'repo-metadata' module, which itself contains information on each # repository in the KDE build system (though currently not # dependencies). The module is meant to be held by the # # Parameters: # ctx - the for this script execution. sub getProjectMetadataModule { my $ctx = assert_isa(shift, 'ksb::BuildContext'); return _createMetadataModule($ctx, 'sysadmin/repo-metadata'); } # Function: _expandModuleCandidates # # A class method which goes through the modules in our search list (assumed to # be found in the kde-projects XML database) and expands them into their # equivalent git modules, and returns the fully expanded list. Non kde-projects # modules cause an error, as do modules that do not exist at all within the # database. # # *Note*: Before calling this function, the kde-projects database itself must # have been downloaded first. Additionally a handling build support # metadata must be included at the beginning of the module list, see # getMetadataModule() for details. # # *Note*: Any modules that are part of a module-set requiring a specific # branch, that don't have that branch, are also elided with only a debug # message. This allows for building older branches of KDE even when newer # modules are eventually put into the database. # # Parameters: # ctx - The in use. # moduleSearchItem - The search description to expand in ksb::Modules. See # _projectPathMatchesWildcardSearch for a description of the syntax. # # Returns: # @modules - List of expanded git . # # Throws: # Runtime - if the kde-projects database was required but couldn't be # downloaded or read. # Runtime - if the git-desired-protocol is unsupported. # Runtime - if an "assumed" kde-projects module was not actually one. sub _expandModuleCandidates { my $self = assert_isa(shift, 'ksb::ModuleSet::KDEProjects'); my $ctx = assert_isa(shift, 'ksb::BuildContext'); my $moduleSearchItem = shift; my $srcdir = $ctx->getSourceDir(); my $xmlReader = $ctx->getProjectDataReader(); my @allXmlResults = $xmlReader->getModulesForProject($moduleSearchItem); croak_runtime ("Unknown KDE project: $moduleSearchItem") unless @allXmlResults; # It's possible to match modules which are marked as inactive on # projects.kde.org, elide those. my @xmlResults = grep { $_->{'active'} } (@allXmlResults); # Base project metadata doesn't include information on which git branches # are available, so we can't elide modules until they're actually downloaded # and available (unless it's available on api.kde.org ?) # # Bug 307694 # my $moduleSetBranch = $self->{'options'}->{'branch'} // ''; # if ($moduleSetBranch && !exists $self->{'options'}->{'tag'}) { # debug ("Filtering kde-projects modules that don't have a $moduleSetBranch branch"); # @xmlResults = grep { # list_has($_->{'branches'}, $moduleSetBranch) # } (@xmlResults); # } if (!@xmlResults) { warning (" y[b[*] Module y[$moduleSearchItem] is apparently XML-based, but contains no\n" . "active modules to build!"); my $count = scalar @allXmlResults; if ($count > 0) { warning ("\tAlthough no active modules are available, there were\n" . "\t$count inactive modules. Perhaps the git modules are not ready?"); } } # Setup module options. my @moduleList; my @ignoreList = $self->modulesToIgnore(); foreach (@xmlResults) { my $result = $_; my $repo = $result->{'repo'}; # Prefer kde: alias to normal clone URL. $repo =~ s(^git://anongit\.kde\.org/)(kde:); my $newModule = ksb::Module->new($ctx, $result->{'name'}); $self->_initializeNewModule($newModule); $newModule->setOption('repository', $repo); $newModule->setOption('#xml-full-path', $result->{'fullName'}); - $newModule->setOption('#branch:stable', $result->{'branch:stable'}); + $newModule->setOption('#branch:stable', undef); + $newModule->setOption('#found-by', $result->{found_by}); $newModule->setScmType('proj'); - my $tarball = $result->{'tarball'}; - $newModule->setOption('#snapshot-tarball', $tarball) if $tarball; - if (none_true( map { ksb::KDEXMLReader::_projectPathMatchesWildcardSearch( $result->{'fullName'}, $_ ) } (@ignoreList))) { push @moduleList, $newModule; } else { debug ("--- Ignoring matched active module $newModule in module set " . $self->name()); } }; return @moduleList; } # This function should be called after options are read and build metadata is # available in order to convert this module set to a list of ksb::Module. # Any modules ignored by this module set are excluded from the returned list. # The modules returned have not been added to the build context. sub convertToModules { my ($self, $ctx) = @_; my @moduleList; # module names converted to ksb::Module objects. my %foundModules; # Setup default options for each module # Extraction of relevant XML modules will be handled immediately after # this phase of execution. for my $moduleItem ($self->modulesToFind()) { # We might have already grabbed the right module recursively. next if exists $foundModules{$moduleItem}; # eval in case the XML processor throws an exception. undef $@; my @candidateModules = eval { $self->_expandModuleCandidates($ctx, $moduleItem); }; if ($@) { die $@ if had_an_exception(); # Forward exception objects up croak_runtime("The XML for the KDE Project database could not be understood: $@"); } my @moduleNames = map { $_->name() } @candidateModules; @foundModules{@moduleNames} = (1) x @moduleNames; push @moduleList, @candidateModules; } if (not scalar @moduleList) { warning ("No modules were defined for the module-set " . $self->name()); warning ("You should use the g[b[use-modules] option to make the module-set useful."); } return @moduleList; } 1;