[kdesrc-build/make_it_mojo] /: Merge branch 'master' into make_it_mojo
Michael Pyne
null at kde.org
Sun Jun 16 02:30:07 BST 2019
Git commit c7de57d6db8c9191501bdd0b32dbe3f741f6c14d by Michael Pyne.
Committed on 15/06/2019 at 19:51.
Pushed by mpyne into branch 'make_it_mojo'.
Merge branch 'master' into make_it_mojo
This does nothing more than startup and immediately fail with a
"kdesrc-build -p" at this point, but this hasn't ended up as broken as
I'd been afraid it would by now.
Conflicts:
modules/ksb/Application.pm
t/read-module-options.t
t/smoke/bug-394497-ignore-dep-module.t
vim/syntax/kdesrc-buildrc.vim
M +6 -0 CMakeLists.txt
M +72 -2 doc/index.docbook
M +56 -0 doc/man-kdesrc-build.1.docbook
M +19 -5 kdesrc-build
M +147 -33 modules/ksb/Application.pm
M +2 -1 modules/ksb/BuildSystem.pm
M +23 -1 modules/ksb/Debug.pm
M +31 -18 modules/ksb/Module.pm
M +59 -35 modules/ksb/Updater/Git.pm
M +105 -0 modules/ksb/Updater/Qt5.pm
M +0 -0 t/smoke/branch-time-based.t
M +31 -9 t/smoke/bug-394497-ignore-dep-module.t
M +0 -0 t/smoke/bug-395627-keep-cmake-prefix.t
M +0 -0 t/smoke/bug-402509-masked-cmdline-opts.t
M +0 -0 t/smoke/install-and-run.t
M +84 -0 t/smoke/read-module-options.t
M +5 -2 t/smoke/set-module-option-value.t
M +3 -2 vim/syntax/kdesrc-buildrc.vim
https://invent.kde.org/kde/kdesrc-build/commit/c7de57d6db8c9191501bdd0b32dbe3f741f6c14d
diff --cc CMakeLists.txt
index 8ac976c,171890e..4d3c56f
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@@ -71,8 -73,14 +73,9 @@@ if (KDESRC_BUILD_INSTALL_MODULES
install(FILES
modules/ksb/ModuleSet/KDEProjects.pm
modules/ksb/ModuleSet/Null.pm
+ modules/ksb/ModuleSet/Qt.pm
DESTINATION ${KDESRC_BUILD_MODULE_INSTALL_PREFIX}/ksb/ModuleSet)
- install(FILES
- modules/ksb/IPC/Null.pm
- modules/ksb/IPC/Pipe.pm
- DESTINATION ${KDESRC_BUILD_MODULE_INSTALL_PREFIX}/ksb/IPC)
-
install(FILES
modules/ksb/Updater/Bzr.pm
modules/ksb/Updater/Git.pm
diff --cc kdesrc-build
index f555f8e,e2b79eb..fd81733
--- a/kdesrc-build
+++ b/kdesrc-build
@@@ -72,7 -72,7 +72,7 @@@ use lib "$modPath"; # Make ksb:: module
sub dumpError
{
-- my $err = $@;
++ my $err = shift;
open my $fh, '>>', "error-$$.log" or return;
my $time = localtime;
say $fh $time;
@@@ -92,7 -92,7 +92,7 @@@ eval
};
if ($@) {
-- dumpError();
++ dumpError($@);
say STDERR <<DONE;
* Unable to even load the simplistic initial setup support for some reason??
@@@ -127,10 -122,10 +127,12 @@@ eval
};
if ($@) {
-- dumpError();
++ my $err = $@;
++ dumpError($err);
say STDERR <<DONE;
Couldn't load the base platform for kdesrc-build!
--More detail might be available in error-$$.log
++
++ $err
DONE
if (! -e "kdesrc-buildrc" && ! -e "$ENV{HOME}/.kdesrc-buildrc") {
@@@ -399,28 -361,13 +401,40 @@@ END
# Use some exception handling to avoid ucky error messages
eval
{
- $app = ksb::Application->new(@ARGV);
+ my $app = web::BackendServer->new(@ARGV);
+ my $ui = ksb::UserInterface::TTY->new($app);
+
++ my %debugFlags = %{$app->ksb->{debugFlags} // {}};
++ if (%debugFlags) {
++ if ($debugFlags{'dependency-tree'}) {
++ say "Dumping dependency tree";
++ }
++ elsif ($debugFlags{'list-build'}) {
++ say "Dumping build list";
++ }
+
- push @atexit_subs, sub { $app->finish(99) };
- my $result = $app->runAllModulePhases();
++ exit 0;
++ }
++
+ push @atexit_subs, sub { $app->ksb->finish(99) };
+
+ # TODO: Reimplement --print-modules, --query modes, which wouldn't go through ->start
+ my $result = $ui->start();
@atexit_subs = (); # Clear exit handlers
- $app->finish($result);
+
+ # env driver is just the ~/.config/kde-env-*.sh, session driver is that + ~/.xsession
+ my $ctx = $app->context;
+ if ($ctx->getOption('install-environment-driver') ||
+ $ctx->getOption('install-session-driver'))
+ {
+ ksb::Application::_installCustomSessionDriver($ctx);
+ }
+
+ # Exits the script
+ my $logdir = $app->context()->getLogDir();
+ note ("Your logs are saved in y[$logdir]");
+ exit $result;
};
if (my $err = $@)
diff --cc modules/ksb/Application.pm
index a9c6e56,1a3d579..b54cf69
--- a/modules/ksb/Application.pm
+++ b/modules/ksb/Application.pm
@@@ -20,10 -20,12 +20,11 @@@ use ksb::Module
use ksb::ModuleResolver 0.20;
use ksb::ModuleSet 0.20;
use ksb::ModuleSet::KDEProjects;
+ use ksb::ModuleSet::Qt;
use ksb::OSSupport;
+use ksb::PromiseChain;
use ksb::RecursiveFH;
use ksb::DependencyResolver 0.20;
-use ksb::IPC::Pipe 0.20;
-use ksb::IPC::Null;
use ksb::Updater::Git;
use ksb::Version qw(scriptVersion);
@@@ -335,13 -334,52 +337,56 @@@ DON
= values %auxOptions;
}
+ sub _yieldModuleDependencyTreeEntry
+ {
+ my ($nodeInfo, $module, $context) = @_;
+
+ my $depth = $nodeInfo->{depth};
+ my $index = $nodeInfo->{idx};
+ my $count = $nodeInfo->{count};
+ my $build = $nodeInfo->{build};
+ my $currentItem = $nodeInfo->{currentItem};
+ my $currentBranch = $nodeInfo->{currentBranch};
+ my $parentItem = $nodeInfo->{parentItem};
+ my $parentBranch = $nodeInfo->{parentBranch};
+
+ my $buildStatus = $build ? 'built' : 'not built';
+ my $statusInfo = $currentBranch ? "($buildStatus: $currentBranch)" : "($buildStatus)";
+
+ my $connectorStack = $context->{stack};
+
+
+ my $prefix = pop(@$connectorStack);
+
+ while($context->{depth} > $depth) {
+ $prefix = pop(@$connectorStack);
+ --($context->{depth});
+ }
+
+ push(@$connectorStack, $prefix);
+
+ my $connector;
+
+ if ($depth == 0) {
+ $connector = $prefix . ' ── ';
+ push(@$connectorStack, $prefix . (' ' x 4));
+ }
+ else {
+ $connector = $prefix . ($index == $count ? '└── ': '├── ');
+ push(@$connectorStack, $prefix . ($index == $count ? ' ' x 4: '│ '));
+ }
+
+ $context->{depth} = $depth + 1;
+ $context->{report}($connector . $currentItem . ' ' . $statusInfo);
+ }
+
-# Generates the build context and module list based on the command line options
-# and module selectors provided, resolves dependencies on those modules if needed,
-# filters out ignored or skipped modules, and sets up the module factory.
+# Generates the build context, builds various module, dependency and branch
+# group resolvers, and splits up the provided option/selector mix read from
+# cmdline into selectors (returned to caller, if any) and pre-built context and
+# resolvers.
+#
+# Use "modulesFromSelectors" to further generate the list of ksb::Modules in
+# dependency order.
#
# After this function is called all module set selectors will have been
# expanded, and we will have downloaded kde-projects metadata.
@@@ -369,6 -407,6 +414,14 @@@ sub establishContex
my %ignoredSelectors =
map { $_, 1 } @{$cmdlineGlobalOptions->{'ignore-modules'}};
++ # Set aside debug-related flags for kdesrc-build CLI driver to handle
++ my @debugFlags = qw(dependency-tree list-build);
++ $self->{debugFlags} = {
++ map { ($_, 1) }
++ grep { defined $cmdlineGlobalOptions->{$_} }
++ (@debugFlags)
++ };
++
my @startProgramAndArgs = @{$cmdlineGlobalOptions->{'start-program'}};
delete @{$cmdlineGlobalOptions}{qw/ignore-modules start-program/};
@@@ -445,12 -486,10 +498,9 @@@
# We also might have cmdline "selectors" to determine which modules or
# module-sets to choose. First let's select module sets, and expand them.
-- my @globalCmdlineArgs = keys %{$cmdlineGlobalOptions};
-- my $commandLineModules = scalar @selectors;
--
- my $moduleResolver = ksb::ModuleResolver->new($ctx);
+ my $moduleResolver
+ = $self->{module_resolver}
+ = ksb::ModuleResolver->new($ctx);
$moduleResolver->setCmdlineOptions($cmdlineOptions);
$moduleResolver->setDeferredOptions($deferredOptions);
$moduleResolver->setInputModulesAndOptions(\@optionModulesAndSets);
@@@ -512,13 -534,53 +562,50 @@@ sub modulesFromSelector
(!defined $branch or $branch); # This is the actual test
} (@modules);
- @modules = $self->_resolveModuleDependencies(@modules);
+ my $moduleGraph = $self->_resolveModuleDependencyGraph(@modules);
+
+ if (!$moduleGraph || !exists $moduleGraph->{graph}) {
+ croak_runtime("Failed to resolve dependency graph");
+ }
+
- if (exists $cmdlineGlobalOptions->{'dependency-tree'}) {
++ if (exists $self->{debugFlags}->{'dependency-tree'}) {
+ my $depTreeCtx = {
+ stack => [''],
+ depth => 0,
+ report => sub {
+ print(@_, "\n");
+ }
+ };
+ ksb::DependencyResolver::walkModuleDependencyTrees(
+ $moduleGraph->{graph},
+ \&_yieldModuleDependencyTreeEntry,
+ $depTreeCtx,
+ @modules
+ );
+ return;
+ }
+
+ @modules = ksb::DependencyResolver::sortModulesIntoBuildOrder(
+ $moduleGraph->{graph}
+ );
# Filter --resume-foo options. This might be a second pass, but that should
# be OK since there's nothing different going on from the first pass (in
# resolveSelectorsIntoModules) in that event.
@modules = _applyModuleFilters($ctx, @modules);
- # Check for ignored modules (post-expansion)
- @modules = grep { ! exists $ignoredSelectors{$_->name()} } @modules;
-
- if(exists $cmdlineGlobalOptions->{'list-build'}) {
++ if(exists $self->{debugFlags}->{'list-build'}) {
+ for my $module (@modules) {
+ my $branch = ksb::DependencyResolver::_getBranchOf($module);
+ print(' ── ', $module->name());
+ if($branch) {
+ print(' : ', $branch);
+ }
+ print("\n");
+ }
+ return;
+ }
+
return @modules;
}
@@@ -596,14 -658,8 +683,14 @@@ sub _resolveModuleDependencyGrap
my $metadataModule = $ctx->getKDEDependenciesMetadataModule();
my @modules = @_;
- @modules = eval {
+ my $graph = eval {
- my $dependencyResolver = ksb::DependencyResolver->new($self->{module_factory});
+ my $moduleResolver = $self->{module_resolver};
+ my $dependencyResolver = ksb::DependencyResolver->new(sub {
+ # Maps module names (what dep resolver has) to built ksb::Modules
+ # (which we need), needs to include all option handling (cmdline,
+ # rc-file, module-sets, etc)
+ return $moduleResolver->resolveModuleIfPresent(shift);
+ });
my $branchGroup = $ctx->effectiveBranchGroup();
for my $file ('dependency-data-common', "dependency-data-$branchGroup")
@@@ -622,21 -677,36 +708,39 @@@
};
if ($@) {
- warning (" r[b[*] Problems encountered trying to sort modules into correct order:");
+ warning (" r[b[*] Problems encountered trying to determing correct module graph:");
warning (" r[b[*] $@");
warning (" r[b[*] Will attempt to continue.");
+
+ $graph = {
+ graph => undef,
+ syntaxErrors => 0,
+ cycles => 0,
+ trivialCycles => 0,
+ pathErrors => 0,
+ branchErrors => 0,
+ exception => $@
+ };
+ }
+ else {
+ if (!$graph->{graph}) {
+ warning (" r[b[*] Unable to determine correct module graph");
+ warning (" r[b[*] Will attempt to continue.");
+ }
}
- return @modules;
+ $graph->{exception} = undef;
+
+ return $graph;
}
-# Runs all update, build, install, etc. phases. Basically this *is* the
-# script.
-# The metadata module must already have performed its update by this point.
-sub runAllModulePhases
+# Similar to the old interactive runAllModulePhases. Actually performs the
+# build for the modules selected by setModulesToProcess.
+#
+# Returns a Mojo::Promise that must be waited on. The promise resolves to
+# return a single success/failure result; use the event handler for now to get
+# more detail during a build.
+sub startHeadlessBuild
{
my $self = shift;
my $ctx = $self->context();
diff --cc modules/ksb/BuildSystem.pm
index 548166a,d67a79e..89e11f7
--- a/modules/ksb/BuildSystem.pm
+++ b/modules/ksb/BuildSystem.pm
@@@ -132,10 -140,13 +133,10 @@@ sub buildInterna
target => undef,
message => 'Compiling...',
'make-options' => [
- split(' ', $self->module()->getOption('make-options')),
+ split(' ', $self->module()->getOption($optionsName)),
],
logbase => 'build',
- subdirs => [
- split(' ', $self->module()->getOption("checkout-only"))
- ],
- })->{was_successful};
+ });
}
# Return value style: boolean
diff --cc modules/ksb/Debug.pm
index 11a31b8,978dd3b..229c751
--- a/modules/ksb/Debug.pm
+++ b/modules/ksb/Debug.pm
@@@ -6,11 -6,9 +6,11 @@@ use strict
use warnings;
use 5.014;
+use Storable qw(freeze);
+
use Exporter qw(import); # Steal Exporter's import method
our @EXPORT = qw(debug pretending debugging whisper
- note info warning error pretend);
+ note info warning error pretend ksb_debug_inspect);
our @EXPORT_OK = qw(colorize);
# Debugging level constants.
diff --cc modules/ksb/Module.pm
index 0279cbe,c53e6d5..2d9b64b
--- a/modules/ksb/Module.pm
+++ b/modules/ksb/Module.pm
@@@ -259,9 -256,10 +262,10 @@@ sub setScmTyp
when('git') { $newType = ksb::Updater::Git->new($self); }
when('proj') { $newType = ksb::Updater::KDEProject->new($self); }
when('metadata') { $newType = ksb::Updater::KDEProjectMetadata->new($self); }
- when('l10n') { $newType = ksb::l10nSystem->new($self); }
+# when('l10n') { $newType = ksb::l10nSystem->new($self); }
when('svn') { $newType = ksb::Updater::Svn->new($self); }
when('bzr') { $newType = ksb::Updater::Bzr->new($self); }
+ when('qt5') { $newType = ksb::Updater::Qt5->new($self); }
default { $newType = undef; }
}
diff --cc modules/ksb/Updater/Qt5.pm
index 0000000,ef6df13..87662bd
mode 000000,100644..100644
--- a/modules/ksb/Updater/Qt5.pm
+++ b/modules/ksb/Updater/Qt5.pm
@@@ -1,0 -1,106 +1,105 @@@
+ package ksb::Updater::Qt5 0.10;
+
+ # Handles updating Qt 5 source code. Requires git but uses Qt 5's dedicated
+ # 'init-repository' script to keep the source up to date and coherent.
+
+ use strict;
+ use warnings;
+ use 5.014;
+
+ use parent qw(ksb::Updater::Git);
+
+ use ksb::BuildException;
+ use ksb::Debug;
-use ksb::IPC::Null;
+ use ksb::Util;
+
+ sub name
+ {
+ return 'qt5';
+ }
+
+ # Handles calling init-repository to clone or update the appropriate Qt 5
+ # submodules.
+ #
+ # Returns number of commits updated (or rather, will...)
+ sub _updateRepository
+ {
+ my $self = assert_isa(shift, __PACKAGE__);
+
+ my $module = $self->module();
+ my $srcdir = $module->fullpath('source');
+
+ if (!pretending() && (! -e "$srcdir/init-repository" || ! -x _)) {
+ croak_runtime ("The Qt 5 repository update script could not be found, or is not executable!");
+ }
+
+ p_chdir($srcdir);
+
+ # See https://wiki.qt.io/Building_Qt_5_from_Git#Getting_the_source_code for
+ # why we skip web engine by default. As of 2019-01-12 it is only used for
+ # PIM or optionally within Plasma
+ my @modules = split(' ', $module->getOption('use-qt5-modules'));
+ push @modules, qw(default -qtwebengine)
+ unless @modules;
+
+ my $subset_arg = join(',', @modules);
+
+ # -f forces a re-update if necessary
+ my @command = ("$srcdir/init-repository", '-f', "--module-subset=$subset_arg");
+ note ("\tUsing Qt 5 modules: ", join(', ', @modules));
+
+ if (0 != log_command($module, 'init-repository', \@command)) {
+ croak_runtime ("Couldn't update Qt 5 repository submodules!");
+ }
+
+ return 1; # TODO: Count commits
+ }
+
+ # Updates an existing Qt5 super module checkout.
+ # Throws exceptions on failure, otherwise returns number of commits updated
+ # OVERRIDE from super class
+ sub updateExistingClone
+ {
+ my $self = assert_isa(shift, __PACKAGE__);
+
+ # Update init-repository and the shell of the super module itself.
+ my $count = $self->SUPER::updateExistingClone();
+
+ # updateRepository has init-repository work to update the source
+ return $count + $self->_updateRepository();
+ }
+
+ # Either performs the initial checkout or updates the current git checkout
+ # for git-using modules, as appropriate.
+ #
+ # If errors are encountered, an exception is raised.
+ #
+ # Returns the number of *commits* affected.
+ # OVERRIDE from super class
+ sub updateCheckout
+ {
+ my $self = assert_isa(shift, __PACKAGE__);
+ my $module = $self->module();
+ my $srcdir = $module->fullpath('source');
+
+ if (-d "$srcdir/.git") {
+ # Note that this function will throw an exception on failure.
+ return $self->updateExistingClone();
+ }
+ else {
+ $self->_verifySafeToCloneIntoSourceDir($module, $srcdir);
+
+ $self->_clone($module->getOption('repository'));
+
+ note ("\tQt update script is installed, downloading remainder of Qt");
+ note ("\tb[y[THIS WILL TAKE SOME TIME]");
+
+ # With the supermodule cloned, we then need to call into
+ # init-repository to have it complete the checkout.
+ return $self->_updateRepository(); # num commits
+ }
+
+ return 0; # num commits
+ }
+
+ 1;
diff --cc t/smoke/read-module-options.t
index 0000000,b821f25..87058f7
mode 000000,100644..100644
--- a/t/smoke/read-module-options.t
+++ b/t/smoke/read-module-options.t
@@@ -1,0 -1,82 +1,84 @@@
+ use 5.014;
+ use strict;
+ use warnings;
+
+ # Test basic option reading from rc-files
+
+ use Test::More;
+
+ package ksb::test {
+ use Carp qw(confess cluck);
+ use Test::More;
+
+ my %inspectors = (
+ log_command => \&_inspect_log_command,
+ );
+
+ our @CMD;
+ our %OPTS;
+
+ sub inspect {
+ my $mod = shift;
+ my $sub = $inspectors{$mod} or return;
+ goto $sub;
+ };
+
+ sub _inspect_log_command
+ {
+ my ($module, $filename, $argRef, $optionsRef) = @_;
+ confess "No arg to module" unless $argRef;
+ my @command = @{$argRef};
+ if (grep { $_ eq 'cmake' } @command) {
+ @CMD = @command;
+ %OPTS = %{$optionsRef};
+ }
+ };
+
+ 1;
+ };
+
+ use ksb::Application;
+ use ksb::Util qw(trimmed);
+
-my $app = ksb::Application->new(qw(--pretend --rc-file t/data/sample-rc/kdesrc-buildrc));
-my @moduleList = @{$app->{modules}};
++my @args = qw(--pretend --rc-file t/data/sample-rc/kdesrc-buildrc);
++my $app = ksb::Application->new;
++my @selectors = $app->establishContext(@args);
++my @moduleList = $app->modulesFromSelectors(@selectors);
+
+ is(scalar @moduleList, 4, 'Right number of modules');
+ is($moduleList[0]->name(), 'module2', 'Right module name');
+
+ my $scm = $moduleList[0]->scm();
+ isa_ok($scm, 'ksb::Updater::Git');
+
+ my ($branch, $type) = $scm->_determinePreferredCheckoutSource();
+
+ is($branch, 'refs/tags/fake-tag5', 'Right tag name');
+ is($type, 'tag', 'Result came back as a tag');
+
+ is($moduleList[2]->name(), 'setmod2', 'Right module name from module-set');
+ ($branch, $type) = $moduleList[2]->scm()->_determinePreferredCheckoutSource();
+
+ is($branch, 'refs/tags/tag-setmod2', 'Right tag name (options block)');
+ is($type, 'tag', 'options block came back as tag');
+
+ #
+ # Test some of the option parsing indirectly by seeing how the value is input
+ # into build system.
+ #
+
+ # Override auto-detection since no source is downloaded
+ $moduleList[2]->setOption('override-build-system', 'kde');
+
+ # Should do nothing in --pretend
+ ok($moduleList[2]->setupBuildSystem(), 'setup fake build system');
+
+ ok(@ksb::test::CMD, 'log_command cmake was called');
+ is($ksb::test::CMD[2], '-DCMAKE_BUILD_TYPE=a b', 'CMake options can be quoted');
+ is($ksb::test::CMD[3], 'bar=c', 'CMake option quoting does not eat all options');
+ is($ksb::test::CMD[-1], "-DCMAKE_INSTALL_PREFIX=$ENV{HOME}/kde", 'Prefix is passed to cmake');
+
+ # See https://phabricator.kde.org/D18165
+ is($moduleList[1]->getOption('cxxflags'), '', 'empty cxxflags renders with no whitespace in module');
+
+ done_testing();
diff --cc vim/syntax/kdesrc-buildrc.vim
index c40c965,b55fdcc..d26170a
--- a/vim/syntax/kdesrc-buildrc.vim
+++ b/vim/syntax/kdesrc-buildrc.vim
@@@ -36,9 -36,10 +36,10 @@@ setlocal iskeyword+=
" Keywords
syn keyword ksbrcOption contained skipwhite nextgroup=ksbrcStringValue
- \ binpath branch build-dir checkout-only cmake-options configure-flags
+ \ binpath branch build-dir cmake-options configure-flags
\ custom-build-command cxxflags dest-dir do-not-compile kdedir
\ libpath log-dir make-install-prefix make-options module-base-path
+ \ ninja-options
\ override-build-system override-url prefix qtdir repository
\ revision source-dir svn-server tag remove-after-install
\ qmake-options git-user
More information about the kde-doc-english
mailing list