diff --git a/modules/ksb/KDEProjectsReader.pm b/modules/ksb/KDEProjectsReader.pm index f1a9f99..451ab59 100644 --- a/modules/ksb/KDEProjectsReader.pm +++ b/modules/ksb/KDEProjectsReader.pm @@ -1,217 +1,231 @@ package ksb::KDEProjectsReader 0.50; # Class: KDEProjectsReader # # Enumerates and provides basic metadata of KDE projects, based on # the YAML metadata included in sysadmin/repo-management. use strict; use warnings; use 5.014; use File::Find; +use ksb::BuildException; + sub _verifyYAMLModuleLoaded { # Load YAML-reading module if available without causing compile error if it # isn't. Note that YAML::Tiny and YAML do not work since some metadata files # use features it doesn't support my @YAML_Opts = qw(Dump Load LoadFile); my @YAML_Mods = qw(YAML::XS YAML::Syck YAML::PP); my $success = 0; foreach my $mod (@YAML_Mods) { $success ||= eval "require $mod; $mod->import(\@YAML_Opts); 1;"; last if $success; } if (!$success) { die "Unable to load one of " . join(', ', @YAML_Mods) . " modules, one of which is needed to handle KDE project data."; } } # Method: new # # Constructs a new KDEProjectsReader. This doesn't contradict any part of the class # documentation which claims this class is a singleton however. This should be # called as a method (e.g. KDEProjectsReader->new(...)). # # Parameters: # $projectMetadataModule - ksb::Module reference to the repo-metadata module. sub new { my $class = shift; my $projectMetadataModule = shift; _verifyYAMLModuleLoaded(); my $self = { # Maps short names to repo info blocks repositories => { } }; $self = bless ($self, $class); $self->_readProjectData($projectMetadataModule); return $self; } # The 'main' method for this class. Reads in *all* KDE projects and notes # their details for later queries. # Be careful, can throw exceptions. sub _readProjectData { my ($self, $projectMetadataModule) = @_; my $srcdir = $projectMetadataModule->fullpath('source'); - my $file_searched = 0; - File::Find::find(sub { - $self->_readYAML($_) if $_ eq 'metadata.yaml'; - $file_searched = 1; - }, "$srcdir/projects"); + croak_runtime("No such source directory $srcdir!") + unless -d $srcdir; + + my $files_searched = 0; + + File::Find::find({ + wanted => sub { + if ($_ eq 'metadata.yaml') { + $self->_readYAML($_); + $files_searched++; + } + }, + follow => 1, + }, "$srcdir/projects"); + + croak_runtime("Failed to find KDE project entries from $srcdir!") + unless $files_searched > 0; } sub _readYAML { my ($self, $filename) = @_; my $proj_data = LoadFile($filename); if (!$proj_data->{repoactive} || !$proj_data->{hasrepo} || # # previously there used to be a check on 'type' which # only caught two specific cases: everything else was # already covered by the hasrepo/repoactive checks. # # replace the type assertion with a specific check on the # projects that would otherwise have been ignored # ($proj_data->{projectpath} eq 'kde-build-metadata' || $proj_data->{projectpath} eq 'repo-management')) { return; }; my $repoPath = $proj_data->{repopath}; my $repoName = $proj_data->{identifier} // $repoPath; my $curRepository = { 'fullName' => $proj_data->{projectpath}, 'repo' => "kde:$repoPath", 'name' => $repoName, 'active' => !!$proj_data->{repoactive}, 'found_by' => 'direct', # can be changed in getModulesForProject }; $self->{repositories}->{$repoName} = $curRepository; } # Note on $proj: A /-separated path is fine, in which case we look # for the right-most part of the full path which matches all of searchProject. # e.g. kde/kdebase/kde-runtime would be matched by a proj of either # "kdebase/kde-runtime" or simply "kde-runtime". sub getModulesForProject { my ($self, $proj) = @_; my $repositoryRef = $self->{repositories}; my @results; my $findResults = sub { my @matchList = grep { _projectPathMatchesWildcardSearch( $repositoryRef->{$_}->{'fullName'}, $proj) } (sort keys %{$repositoryRef}); if ($proj =~ m/\*/) { $repositoryRef->{$_}->{found_by} = 'wildcard' foreach @matchList; } push @results, @matchList; }; # Wildcard matches happen as specified if asked for. # Non-wildcard matches have an implicit "$proj/*" search as well for # compatibility with previous use-modules # Project specifiers ending in .git are forced to be non-wildcarded. if ($proj !~ /\*/ && $proj !~ /\.git$/) { # We have to do a search to account for over-specified module names # like phonon/phonon $findResults->(); # Now setup for a wildcard search to find things like kde/kdelibs/baloo # if just 'kdelibs' is asked for. $proj .= '/*'; } $proj =~ s/\.git$//; # If still no wildcard and no '/' then we can use direct lookup by module # name. if ($proj !~ /\*/ && $proj !~ /\// && exists $repositoryRef->{$proj}) { push @results, $proj; } else { $findResults->(); } return @{$repositoryRef}{@results}; } # Utility subroutine, returns true if the given kde-project full path (e.g. # kde/kdelibs/nepomuk-core) matches the given search item. # # The search item itself is based on path-components. Each path component in # the search item must be present in the equivalent path component in the # module's project path for a match. A '*' in a path component position for the # search item matches any project path component. # # Finally, the search is pinned to search for a common suffix. E.g. a search # item of 'kdelibs' would match a project path of 'kde/kdelibs' but not # 'kde/kdelibs/nepomuk-core'. However 'kdelibs/*' would match # 'kde/kdelibs/nepomuk-core'. # # First parameter is the full project path from the kde-projects database. # Second parameter is the search item. # Returns true if they match, false otherwise. sub _projectPathMatchesWildcardSearch { my ($projectPath, $searchItem) = @_; my @searchParts = split(m{/}, $searchItem); my @nameStack = split(m{/}, $projectPath); if (scalar @nameStack >= scalar @searchParts) { my $sizeDifference = scalar @nameStack - scalar @searchParts; # We might have to loop if we somehow find the wrong start point for our search. # E.g. looking for a/b/* against a/a/b/c, we'd need to start with the second a. my $i = 0; while ($i <= $sizeDifference) { # Find our common prefix, then ensure the remainder matches item-for-item. for (; $i <= $sizeDifference; $i++) { last if $nameStack[$i] eq $searchParts[0]; } return if $i > $sizeDifference; # Not enough room to find it now # At this point we have synched up nameStack to searchParts, ensure they # match item-for-item. my $found = 1; for (my $j = 0; $found && ($j < @searchParts); $j++) { return 1 if $searchParts[$j] eq '*'; # This always works $found = 0 if $searchParts[$j] ne $nameStack[$i + $j]; } return 1 if $found; # We matched every item to the substring we found. $i++; # Try again } } return; } 1; diff --git a/modules/ksb/ModuleSet/KDEProjects.pm b/modules/ksb/ModuleSet/KDEProjects.pm index 56f5fb2..a394730 100644 --- a/modules/ksb/ModuleSet/KDEProjects.pm +++ b/modules/ksb/ModuleSet/KDEProjects.pm @@ -1,223 +1,226 @@ package ksb::ModuleSet::KDEProjects 0.30; # Class: ModuleSet::KDEProjects # # This represents a collective grouping of modules that share common options, # based on the KDE project repositories. Metadata for that repository is # itself housed in a dedicated KDE.org git repository "sysadmin/repo-metadata", # which this class uses to imbue ksb::Modules generated by this ModuleSet. # # The only changes here are to allow for expanding out module specifications # (except for ignored modules), by using KDEProjectsReader. # # See also: ModuleSet use strict; use warnings; use 5.014; use parent qw(ksb::ModuleSet); no if $] >= 5.018, 'warnings', 'experimental::smartmatch'; use ksb::BuildContext 0.20; use ksb::BuildException; use ksb::Debug; use ksb::KDEProjectsReader 0.50; use ksb::Module; use ksb::Util; sub new { my $self = ksb::ModuleSet::new(@_); $self->{projectsDataReader} = undef; # Will be filled in when we get fh return $self; } # Simple utility subroutine. See List::Util's perldoc sub none_true { ($_ && return 0) for @_; return 1; } sub _createMetadataModule { my ($ctx, $moduleName) = @_; my $metadataModule = ksb::Module->new($ctx, $moduleName =~ s,/,-,r); # Hardcode the results instead of expanding out the project info $metadataModule->setOption('repository', "kde:$moduleName"); $metadataModule->setOption('#xml-full-path', $moduleName); $metadataModule->setOption('#branch:stable', 'master'); $metadataModule->setScmType('metadata'); $metadataModule->setOption('disable-snapshots', 1); $metadataModule->setOption('branch', 'master'); my $moduleSet = ksb::ModuleSet::KDEProjects->new($ctx, ''); $metadataModule->setModuleSet($moduleSet); # Ensure we only ever try to update source, not build. $metadataModule->phases()->phases('update'); return $metadataModule; } # Function: getDependenciesModule # # Static. Returns a that can be used to download the # 'kde-build-metadata' module, which itself contains module dependencies # in the KDE build system. The module is meant to be held by the # # Parameters: # ctx - the for this script execution. sub getDependenciesModule { my $ctx = assert_isa(shift, 'ksb::BuildContext'); return _createMetadataModule($ctx, 'kde-build-metadata'); } # Function: getProjectMetadataModule # # Static. Returns a that can be used to download the # 'repo-metadata' module, which itself contains information on each # repository in the KDE build system (though currently not # dependencies). The module is meant to be held by the # # Parameters: # ctx - the for this script execution. sub getProjectMetadataModule { my $ctx = assert_isa(shift, 'ksb::BuildContext'); return _createMetadataModule($ctx, 'sysadmin/repo-metadata'); } # Function: _expandModuleCandidates # # A class method which goes through the modules in our search list (assumed to # be found in kde-projects), expands them into their equivalent git modules, # and returns the fully expanded list. Non kde-projects modules cause an error, # as do modules that do not exist at all within the database. # # *Note*: Before calling this function, the kde-projects database itself must # have been downloaded first. See getProjectMetadataModule, which ties to the # BuildContext. # # Modules that are part of a module-set requiring a specific branch, that don't # have that branch, are still listed in the return result since there's no way # to tell that the branch won't be there. These should be removed later. # # Parameters: # ctx - The in use. # moduleSearchItem - The search description to expand in ksb::Modules. See # _projectPathMatchesWildcardSearch for a description of the syntax. # # Returns: # @modules - List of expanded git . # # Throws: # Runtime - if the kde-projects database was required but couldn't be # downloaded or read. # Runtime - if the git-desired-protocol is unsupported. # Runtime - if an "assumed" kde-projects module was not actually one. sub _expandModuleCandidates { my $self = assert_isa(shift, 'ksb::ModuleSet::KDEProjects'); my $ctx = assert_isa(shift, 'ksb::BuildContext'); my $moduleSearchItem = shift; + whisper("There are " . (scalar keys %{$ctx->getProjectDataReader()->{repositories}}) . " repos in db."); + my @allModuleResults = $ctx-> getProjectDataReader()-> getModulesForProject($moduleSearchItem); - croak_runtime ("Unknown KDE project: $moduleSearchItem") unless @allModuleResults; + croak_runtime ("Unknown KDE project: $moduleSearchItem") + unless @allModuleResults; # It's possible to match modules which are marked as inactive on # projects.kde.org, elide those. my @activeResults = grep { $_->{'active'} } (@allModuleResults); if (!@activeResults) { warning (" y[b[*] Module y[$moduleSearchItem] is apparently a KDE collection, but contains no\n" . "active modules to build!"); my $count = scalar @allModuleResults; if ($count > 0) { warning ("\tAlthough no active modules are available, there were\n" . "\t$count inactive modules. Perhaps the git modules are not ready?"); } } # Setup module options. my @moduleList; my @ignoreList = $self->modulesToIgnore(); foreach (@activeResults) { my $result = $_; my $newModule = ksb::Module->new($ctx, $result->{'name'}); $self->_initializeNewModule($newModule); $newModule->setOption('repository', $result->{'repo'}); $newModule->setOption('#xml-full-path', $result->{'fullName'}); $newModule->setOption('#branch:stable', undef); $newModule->setOption('#found-by', $result->{found_by}); $newModule->setScmType('proj'); if (none_true( map { ksb::KDEProjectsReader::_projectPathMatchesWildcardSearch( $result->{'fullName'}, $_ ) } (@ignoreList))) { push @moduleList, $newModule; } else { debug ("--- Ignoring matched active module $newModule in module set " . $self->name()); } }; return @moduleList; } # This function should be called after options are read and build metadata is # available in order to convert this module set to a list of ksb::Module. # Any modules ignored by this module set are excluded from the returned list. # The modules returned have not been added to the build context. sub convertToModules { my ($self, $ctx) = @_; my @moduleList; # module names converted to ksb::Module objects. my %foundModules; # Setup default options for each module # Extraction of relevant kde-project modules will be handled immediately # after this phase of execution. for my $moduleItem ($self->modulesToFind()) { # We might have already grabbed the right module recursively. next if exists $foundModules{$moduleItem}; # eval in case the YAML processor throws an exception. undef $@; my @candidateModules = eval { $self->_expandModuleCandidates($ctx, $moduleItem); }; if ($@) { die $@ if had_an_exception(); # Forward exception objects up croak_runtime("The KDE Project database could not be understood: $@"); } my @moduleNames = map { $_->name() } @candidateModules; @foundModules{@moduleNames} = (1) x @moduleNames; push @moduleList, @candidateModules; } if (not scalar @moduleList) { warning ("No modules were defined for the module-set " . $self->name()); warning ("You should use the g[b[use-modules] option to make the module-set useful."); } return @moduleList; } 1;