[kdesrc-build/make_it_mojo] /: Merge remote-tracking branch 'gitlab/master' into make_it_mojo
Michael Pyne
null at kde.org
Sun Sep 1 22:31:20 BST 2019
Git commit c1fb8cbb5faad6bcddbd17a85b1a3a8e8e1d01cd by Michael Pyne.
Committed on 01/09/2019 at 20:30.
Pushed by mpyne into branch 'make_it_mojo'.
Merge remote-tracking branch 'gitlab/master' into make_it_mojo
Main source of difficulty is integrating the ksb::Application changes
for debug-order hinting into the adapted backend/client split flow
paths, but seems to work OK.
TODO: Still need to fix test suite and/or fix broken code caught by
tests, as appropriate.
Conflicts:
CMakeLists.txt
modules/ksb/Application.pm
t/smoke/branch-time-based.t
M +4 -1 CMakeLists.txt
M +25 -4 doc/index.docbook
M +8 -5 doc/man-kdesrc-build.1.docbook
M +1 -1 kdesrc-build
M +92 -17 modules/ksb/Application.pm
M +1 -0 modules/ksb/BuildContext.pm
M +9 -23 modules/ksb/Updater/Git.pm
M +2 -2 t/smoke/branch-time-based.t
https://invent.kde.org/kde/kdesrc-build/commit/c1fb8cbb5faad6bcddbd17a85b1a3a8e8e1d01cd
diff --cc modules/ksb/Application.pm
index b549d41,9fcc617..f5691ca
--- a/modules/ksb/Application.pm
+++ b/modules/ksb/Application.pm
@@@ -22,9 -22,11 +22,10 @@@ use ksb::ModuleSet 0.20
use ksb::ModuleSet::KDEProjects;
use ksb::ModuleSet::Qt;
use ksb::OSSupport;
+use ksb::PromiseChain;
use ksb::RecursiveFH;
use ksb::DependencyResolver 0.20;
+ use ksb::DebugOrderHints;
-use ksb::IPC::Pipe 0.20;
-use ksb::IPC::Null;
use ksb::Updater::Git;
use ksb::Version qw(scriptVersion);
@@@ -67,31 -66,32 +68,41 @@@ sub ne
# Default to colorized output if sending to TTY
ksb::Debug::setColorfulOutput(-t STDOUT);
- my $workLoad = $self->generateModuleList(@options);
- if (!$workLoad->{build}) {
- print "No modules to build, exiting.\n";
- exit 0;
- }
+ return $self;
+}
+
+# Call after establishContext (to read in config file and do one-time metadata
- # reading).
++# reading), but before you call startHeadlessBuild.
+#
- # Need to call this before you call startHeadlessBuild
++# Parameter:
++#
++# - workload, a hashref containing the following entries:
++# {
++# selectedModules: listref with the selected ksb::Modules to build
++# dependencyInfo: reference to a dependency info object created by
++# ksb::DependencyResolver
++# build: a boolean indicating whether to go through with build or not
++# }
+sub setModulesToProcess
+{
- my ($self, @modules) = @_;
- $self->{modules} = \@modules;
++ my ($self, $workLoad) = @_;
+
+ $self->{modules} = $workLoad->{selectedModules};
+ $self->{workLoad} = $workLoad;
- $self->context()->setupOperatingEnvironment(); # i.e. niceness, ulimits, etc.
-
- # After this call, we must run the finish() method
- # to cleanly complete process execution.
- if (!pretending() && !$self->context()->takeLock())
- {
- print "$0 is already running!\n";
- exit 1; # Don't finish(), it's not our lockfile!!
- }
+ $self->context()->addModule($_)
- foreach @modules;
++ foreach @{$self->{modules}};
- # Install signal handlers to ensure that the lockfile gets closed.
- _installSignalHandlers(sub {
- note ("Signal received, terminating.");
- @main::atexit_subs = (); # Remove their finish, doin' it manually
- $self->finish(5);
- });
+ # i.e. niceness, ulimits, etc.
+ $self->context()->setupOperatingEnvironment();
+}
+# Sets the application to be non-interactive, intended to make this suitable as
+# a backend for a Mojolicious-based web server with a separate U/I.
+sub setHeadless
+{
+ my $self = shift;
+ $self->{run_mode} = 'headless';
return $self;
}
@@@ -501,35 -501,10 +511,35 @@@ sub establishContex
$moduleResolver->setInputModulesAndOptions(\@optionModulesAndSets);
$moduleResolver->setIgnoredSelectors([keys %ignoredSelectors]);
- $self->_defineNewModuleFactory($moduleResolver);
+ # The user might only want metadata to update to allow for a later
+ # --pretend run, check for that here.
+ if (exists $cmdlineGlobalOptions->{'metadata-only'}) {
+ return;
+ }
+
+ return @selectors;
+}
+
+# Requires establishContext to have been called first. Converts string-based
+# "selectors" for modules or module-sets into a list of ksb::Modules (only
- # modules, no sets).
++# modules, no sets), and returns associated metadata including dependencies.
+#
+# After this function is called all module set selectors will have been
+# expanded, and we will have downloaded kde-projects metadata.
+#
+# The modules returned must still be added (using setModulesToProcess) to the
+# context if you intend to build. This is a separate step to allow for some
+# introspection prior to making choice to build.
+#
- # Returns: List of Modules to build.
++# Returns: A hashref to a workload object (as described in setModulesToProcess)
+sub modulesFromSelectors
+{
+ my ($self, @selectors) = @_;
+ my $moduleResolver = $self->{module_resolver};
+ my $ctx = $self->context();
my @modules;
- if ($commandLineModules) {
+ if (@selectors) {
@modules = $moduleResolver->resolveSelectorsIntoModules(@selectors);
}
else {
@@@ -569,9 -546,27 +579,17 @@@
croak_runtime("Failed to resolve dependency graph");
}
- if (exists $cmdlineGlobalOptions->{'dependency-tree'}) {
- my $depTreeCtx = {
- stack => [''],
- depth => 0,
- report => sub {
- print(@_, "\n");
- }
- };
- ksb::DependencyResolver::walkModuleDependencyTrees(
- $moduleGraph->{graph},
- \&_yieldModuleDependencyTreeEntry,
- $depTreeCtx,
- @modules
- );
++ # TODO: Implement --dependency-tree
+ if (exists $self->{debugFlags}->{'dependency-tree'}) {
+ # Save for later introspection
+ $self->{debugFlags}->{'dependency-tree'} = $moduleGraph->{graph};
+
+ my $result = {
+ dependencyInfo => $moduleGraph,
+ selectedModules => [],
+ build => 0
+ };
+ return $result;
}
@modules = ksb::DependencyResolver::sortModulesIntoBuildOrder(
@@@ -583,7 -578,33 +601,22 @@@
# resolveSelectorsIntoModules) in that event.
@modules = _applyModuleFilters($ctx, @modules);
- return @modules;
- # Check for ignored modules (post-expansion)
- @modules = grep { ! exists $ignoredSelectors{$_->name()} } @modules;
-
- if(exists $cmdlineGlobalOptions->{'list-build'}) {
- for my $module (@modules) {
- my $branch = ksb::DependencyResolver::_getBranchOf($module);
- print(' ── ', $module->name());
- if($branch) {
- print(' : ', $branch);
- }
- print("\n");
- }
-
++ # TODO: Implement 'list-build' option
++ if(exists $self->{debugFlags}->{'list-build'}) {
+ my $result = {
+ dependencyInfo => $moduleGraph,
+ selectedModules => [],
+ build => 0
+ };
+ return $result;
+ }
+
+ my $result = {
+ dependencyInfo => $moduleGraph,
+ selectedModules => \@modules,
+ build => 1
+ };
+ return $result;
}
# Causes kde-projects metadata to be downloaded (unless --pretend, --no-src, or
@@@ -721,49 -733,125 +754,53 @@@ sub startHeadlessBuil
{
my $self = shift;
my $ctx = $self->context();
- my @modules = $self->modules();
-
- if ($ctx->getOption('print-modules')) {
- for my $m (@modules) {
- say ((" " x ($m->getOption('#dependency-level', 'module') // 0)), "$m");
- }
- return 0; # Abort execution early!
- }
-
- # Add to global module list now that we've filtered everything.
- $ctx->addModule($_) foreach @modules;
-
- my $runMode = $self->runMode();
-
- if ($runMode eq 'query') {
- my $queryMode = $ctx->getOption('query', 'module');
+ $ctx->statusMonitor()->createBuildPlan($ctx);
- # Default to ->getOption as query method.
- # $_[0] is short name for first param.
- my $query = sub { $_[0]->getOption($queryMode) };
- $query = sub { $_[0]->fullpath('source') } if $queryMode eq 'source-dir';
- $query = sub { $_[0]->fullpath('build') } if $queryMode eq 'build-dir';
- $query = sub { $_[0]->installationPath() } if $queryMode eq 'install-dir';
- $query = sub { $_[0]->fullProjectPath() } if $queryMode eq 'project-path';
- $query = sub { ($_[0]->scm()->_determinePreferredCheckoutSource())[0] // '' }
- if $queryMode eq 'branch';
+ my $promiseChain = ksb::PromiseChain->new;
+ my $startPromise = Mojo::Promise->new;
- if (@modules == 1) {
- # No leading module name, just the value
- say $query->($modules[0]);
- }
- else {
- for my $m (@modules) {
- say "$m: ", $query->($m);
- }
- }
+ # These succeed or die outright
+ $startPromise = _handle_updates ($ctx, $promiseChain, $startPromise);
+ $startPromise = _handle_build ($ctx, $promiseChain, $startPromise);
- return 0;
- }
-
- my $result;
+ die "Can't obtain build lock" unless $ctx->takeLock();
- if ($runMode eq 'build')
- {
- # No packages to install, we're in build mode
-
- # What we're going to do is fork another child to perform the source
- # updates while we build. Setup for this first by initializing some
- # shared memory.
- my $ipc = 0;
- my $updateOptsSub = sub {
- my ($k, $v) = @_;
- $ctx->setPersistentOption($k, $v);
- };
-
- if ($ctx->getOption('async'))
- {
- $ipc = ksb::IPC::Pipe->new();
- $ipc->setPersistentOptionHandler($updateOptsSub);
- }
+ # Install signal handlers to ensure that the lockfile gets closed.
+ _installSignalHandlers(sub {
+ @main::atexit_subs = (); # Remove their finish, doin' it manually
+ $self->finish(5);
+ });
- $startPromise->resolve; # allow build to start
- if (!$ipc)
- {
- $ipc = ksb::IPC::Null->new();
- $ipc->setPersistentOptionHandler($updateOptsSub);
++ $startPromise->resolve; # allow build to start once control returned to evt loop
+ my $promise = $promiseChain->makePromiseChain($startPromise)->finally(sub {
+ my @results = @_;
+ my $result = 0; # success, non-zero is failure
- whisper ("Using no IPC mechanism\n");
+ # Must use ! here to make '0 but true' hack work
+ $result = 1 if defined first { !($_->[0] // 1) } @results;
- note ("\n b[<<< Update Process >>>]\n");
- $result = _handle_updates ($ipc, $ctx);
+ $ctx->statusMonitor()->markBuildDone();
+ $ctx->closeLock();
- note (" b[<<< Build Process >>>]\n");
- $result = _handle_build ($ipc, $ctx) || $result;
+ my $failedModules = join(',', map { "$_" } $ctx->listFailedModules());
+ if ($failedModules) {
+ # We don't clear the list of failed modules on success so that
+ # someone can build one or two modules and still use
+ # --rebuild-failures
+ $ctx->setPersistentOption('global', 'last-failed-module-list', $failedModules);
}
- else
- {
- $result = _handle_async_build ($ipc, $ctx);
- $ipc->outputPendingLoggedMessages() if debugging();
- }
- }
- elsif ($runMode eq 'install')
- {
- $result = _handle_install ($ctx);
- }
- elsif ($runMode eq 'uninstall')
- {
- $result = _handle_uninstall ($ctx);
- }
-
- _cleanup_log_directory($ctx) if $ctx->getOption('purge-old-logs');
-
- my $workLoad = $self->workLoad();
- my $dependencyGraph = $workLoad->{dependencyInfo}->{graph};
- _output_failed_module_lists($ctx, $dependencyGraph);
++ # TODO: Anything to do with this info at this point?
++ my $workLoad = $self->workLoad();
++ my $dependencyGraph = $workLoad->{dependencyInfo}->{graph};
+
- # Record all failed modules. Unlike the 'resume-list' option this doesn't
- # include any successfully-built modules in between failures.
- my $failedModules = join(',', map { "$_" } $ctx->listFailedModules());
- if ($failedModules) {
- # We don't clear the list of failed modules on success so that
- # someone can build one or two modules and still use
- # --rebuild-failures
- $ctx->setPersistentOption('global', 'last-failed-module-list', $failedModules);
- }
-
- # env driver is just the ~/.config/kde-env-*.sh, session driver is that + ~/.xsession
- if ($ctx->getOption('install-environment-driver') ||
- $ctx->getOption('install-session-driver'))
- {
- _installCustomSessionDriver($ctx);
- }
-
- my $color = 'g[b[';
- $color = 'r[b[' if $result;
+ $ctx->storePersistentOptions();
+ _cleanup_log_directory($ctx);
- info ("${color}", $result ? ":-(" : ":-)") unless pretending();
+ return $result;
+ });
- return $result;
+ return $promise;
}
# Method: finish
diff --cc modules/ksb/Updater/Git.pm
index 699d143,f8a6fc5..8cb3825
--- a/modules/ksb/Updater/Git.pm
+++ b/modules/ksb/Updater/Git.pm
@@@ -530,32 -538,18 +530,18 @@@ sub stashAndUpdat
my $module = $self->module();
my $date = strftime ("%F-%R", gmtime()); # ISO Date, hh:mm time
- # To find out if we should stash, we just use git diff --quiet, twice to
- # account for the index and the working dir.
- # Note: Don't use safe_system, as the error code is stripped to the exit code
- my $status = pretending() ? 0 : system('git', 'diff', '--quiet');
-
- if ($status == -1 || $status & 127) {
- croak_runtime("$module doesn't appear to be a git module.");
- }
-
- my $needsStash = 0;
- if ($status && !_hasSubmodules()) {
- # There are local changes.
- $needsStash = 1;
- }
- else {
- $status = pretending() ? 0 : system('git', 'diff', '--cached', '--quiet');
- if ($status == -1 || $status & 127) {
- croak_runtime("$module doesn't appear to be a git module.");
- }
- else {
- $needsStash = ($status != 0);
- }
- }
+ # To find out if we should stash, we use git-status in its short mode which
+ # is intended to be scriptable and returns information on both the index
+ # and the working dir in one command.
+ my $status = 1;
+ my $needsStash =
+ !pretending() &&
+ (scalar filter_program_output(
+ undef, # don't filter output
+ qw(git status --short --untracked-files=no))) > 0;
if ($needsStash) {
- info ("\tLocal changes detected, stashing them away...");
+ info ("\tLocal changes detected (will stash for now and then restore)");
$status = log_command($module, 'git-stash-save', [
qw(git stash save --quiet), "kdesrc-build auto-stash at $date",
]);
diff --cc t/smoke/branch-time-based.t
index 6603b23,70663ff..732f09b
--- a/t/smoke/branch-time-based.t
+++ b/t/smoke/branch-time-based.t
@@@ -8,10 -8,8 +8,10 @@@ use Test::More
use ksb::Application;
-my $app = ksb::Application->new(qw(--pretend --rc-file t/data/branch-time-based/kdesrc-buildrc));
-my @moduleList = $app->modules();
+my @args = qw(--pretend --rc-file t/data/branch-time-based/kdesrc-buildrc);
+my $app = ksb::Application->new;
- my @selectors = $app->establishContext(@args);
- my @moduleList = $app->modulesFromSelectors(@selectors);
++my $workload = $app->establishContext(@args);
++my @moduleList = $app->modulesFromSelectors($workload);
is(scalar @moduleList, 3, 'Right number of modules');
More information about the kde-doc-english
mailing list