[sdk/kdesrc-build/make_it_mojo] /: Merge remote-tracking branch 'origin/master' into make_it_mojo
Michael Pyne
null at kde.org
Mon Aug 17 00:01:12 BST 2020
Git commit 4a81e862df9cca72bbd95dcbfe981dc20ae5d725 by Michael Pyne.
Committed on 16/08/2020 at 21:54.
Pushed by mpyne into branch 'make_it_mojo'.
Merge remote-tracking branch 'origin/master' into make_it_mojo
This required touching up some changes that involve the long-removed IPC
handling stuff.
Conflicts:
modules/ksb/Updater/Git.pm
vim/syntax/kdesrc-buildrc.vim
M +68 -2 doc/index.docbook
M +0 -45 modules/ksb/Application.pm
M +5 -0 modules/ksb/BuildContext.pm
M +3 -0 modules/ksb/Module.pm
M +70 -54 modules/ksb/Updater/Git.pm
M +11 -2 modules/ksb/Util.pm
M +2 -1 vim/syntax/kdesrc-buildrc.vim
https://invent.kde.org/sdk/kdesrc-build/commit/4a81e862df9cca72bbd95dcbfe981dc20ae5d725
diff --cc modules/ksb/Module.pm
index ed4ac4b,0998fff..0310ced
--- a/modules/ksb/Module.pm
+++ b/modules/ksb/Module.pm
@@@ -1019,126 -991,6 +1019,129 @@@ sub installationPat
return $path;
}
+# Supports the subprocess handler in runPhase_p by reading messages that were
+# generated by the child process and sent back to us, and dispatching to
+# handler for real-time updates. Messages that can wait until the subprocess is
+# complete are handled separately.
+sub _readAndDispatchInProcessMessages
+{
+ my ($self, $linesRef, $monitor, $phaseName) = @_;
+
+ croak_internal("Failed to read msg from child handler: $@")
+ unless $linesRef;
+
+ if (exists $linesRef->{message}) {
+ $monitor->noteLogEvents("$self", $phaseName, [$linesRef->{message}]);
+ } elsif (exists $linesRef->{progress}) {
+ my ($x, $y) = @{$linesRef->{progress}};
+ $monitor->markPhaseProgress("$self", $phaseName, $x / $y)
+ if $y > 0;
+ } else {
+ croak_internal("Couldn't handle message $linesRef from child.");
+ }
+}
+
+# Runs the given phase in a separate subprocess, using provided sub references.
+# Assumes use of promises for the provided sub references -- if launching the
+# subprocess fails, then a rejected promise is returned in the completion sub
+# reference
+# Returns a promise that yields the return value of the completion sub
+# reference.
+sub runPhase_p
+{
+ my ($self, $phaseName, $blocking_coderef, $completion_coderef) = @_;
+ my $phaseSubs = $self->{builders}->{$phaseName};
+
+ if (ref($phaseSubs) eq 'CODE') {
+ $blocking_coderef //= $phaseSubs;
+ }
+ elsif (ref($phaseSubs) eq 'ARRAY') {
+ ($blocking_coderef, $completion_coderef) = @{$phaseSubs};
+ croak_internal("Missing subs for $phaseName")
+ unless ($blocking_coderef && $completion_coderef);
+ }
+ else {
+ croak_internal("self->builders->{$phaseName} should not be set")
+ if defined $phaseSubs;
+ }
+
+ # Default handler
+ $completion_coderef //= sub {
+ my ($module, $result) = @_;
+ return $result || Mojo::Promise->new->reject;
+ };
+
+ my $ctx = $self->buildContext();
+ my $monitor = $ctx->statusMonitor();
+ my $subprocess = Mojo::IOLoop->subprocess();
+ my $start_time = time;
+
+ $monitor->markPhaseStart($self->name(), $phaseName);
+
+ # Setup progress handler in parent
+ $subprocess->on(progress => sub {
+ my ($subprocess, $progress_data) = @_;
+ $self->_readAndDispatchInProcessMessages($progress_data, $monitor, $phaseName);
+ });
+
+ # Fork the child process and wait in Mojo::IOLoop event loop
+ return $subprocess->run_p(sub {
+ # blocks, runs in separate process
+ $SIG{INT} = sub { POSIX::_exit(EINTR); };
+ $0 = "kdesrc-build[$phaseName]";
+
+ ksb::Debug::setSubprocessOutputHandle($subprocess);
+
+ $self->buildContext->resetEnvironment();
+ $self->setupEnvironment();
+
+ # This coderef should return a hashref: {
+ # was_successful => bool,
+ # ... (other details)
+ # }
+ my $resultRef = $blocking_coderef->($self);
+
+ # We may have changed options and persistent options, feed those back
+ # to the main process.
+ my @bundles_to_copy = qw(build_options persistent_options);
+ my %option_bundle = map { ($_ => $ctx->{$_}->{"$self"}) } @bundles_to_copy;
+
+ return {
+ result => $resultRef->{was_successful},
++ post_msgs => $self->{post_build_msgs},
+ newOptions => \%option_bundle,
+ extras => $resultRef,
+ };
+ })->then(sub {
+ # runs in this process once subprocess is done
+ my $resultsRef = shift;
+ my $promise = Mojo::Promise->new;
+
+ $self->{metrics}->{time_in_phase}->{$phaseName} = time - $start_time;
+
+ # Apply options that may have changed during child proc execution.
+ while (my ($bundle_name, $options) = each %{$resultsRef->{newOptions}}) {
+ $ctx->{$bundle_name}->{"$self"} = $options;
+ }
+
++ push @{$self->{post_build_msgs}}, @{$resultsRef->{post_msgs}};
++
+ # TODO Make this a ->then handler?
+ my $completion_p = $completion_coderef->($self, $resultsRef->{result}, $resultsRef);
+ if ($resultsRef->{result}) {
+ $ctx->markModulePhaseSucceeded($phaseName, $self, $resultsRef->{extras});
+ return Mojo::Promise->resolve($completion_p);
+ } else {
+ $ctx->markModulePhaseFailed($phaseName, $self);
+ return Mojo::Promise->reject ($completion_p);
+ }
+ })->catch(sub {
+ my $err = shift;
+ $ctx->markModulePhaseFailed($phaseName, $self);
+ return Mojo::Promise->new->reject($err);
+ });
+}
+
# Returns a list of any 'post-build' messages that have been set for the module
# to show after the build has ended. These may be messages such as warning of a
# local source conflict that may have scrolled past or similar things the user
diff --cc modules/ksb/Updater/Git.pm
index bb9c75e,2802036..1fe0ecb
--- a/modules/ksb/Updater/Git.pm
+++ b/modules/ksb/Updater/Git.pm
@@@ -546,6 -557,34 +549,30 @@@ sub _splitUr
return ($scheme, $authority, $path, $query, $fragment);
}
+ sub countStash
+ {
+ my $self = assert_isa(shift, 'ksb::Updater::Git');
+ my $module = $self->module();
+ my $description = shift;
+
+ if (-e '.git/refs/stash') {
+ my $count = qx"git rev-list --walk-reflogs --count refs/stash";
+ chomp $count if $count;
+ debug("\tNumber of stashes found for b[$module] is: b[$count]");
+ return $count;
+ } else {
+ debug("\tIt appears there is no stash for b[$module]");
+ return 0;
+ }
+ }
+
-#
-# Wrapper to send a post-build (warning) message via the IPC object.
-# This just takes care of the boilerplate to forward its arguments as message.
-#
+ sub _notifyPostBuildMessage
+ {
+ my $self = assert_isa(shift, 'ksb::Updater::Git');
+ my $module = $self->module();
+ $self->{ipc}->notifyNewPostBuildMessage($module->name(), @_);
+ }
+
# This stashes existing changes if necessary, and then runs a provided
# update routine in order to advance the given module to the desired head.
# Finally, if changes were stashed, they are applied and the stash stack is
@@@ -568,29 -607,52 +595,52 @@@ sub stashAndUpdat
my $updateSub = shift;
my $module = $self->module();
my $date = strftime ("%F-%R", gmtime()); # ISO Date, hh:mm time
+ my $stashName = "kdesrc-build auto-stash at $date";
- # To find out if we should stash, we use git-diff-index which
- # is intended to be scriptable and returns information on both the index
- # and the working dir in one command.
- my $status = 1;
- my $needsStash =
- !pretending() && (system('git', 'diff-index', '--quiet', 'HEAD') >> 8);
-
+ # first, log a snapshot of the git status prior to kdesrc-build taking over the reins in the repo
log_command($module, 'git-status-before-update', [qw(git status)]);
+ my $oldStashCount = $self->countStash();
+
+ #
+ # always stash:
+ # - also stash untracked files because what if upstream started to track them
+ # - we do not stash .gitignore'd files because they may be needed for builds?
+ # on the other hand that leaves a slight risk if upstream altered those (i.e. no longer truly .gitignore'd)
+ #
+ info ("\tStashing local changes if any...");
+ my $status = 0;
+ $status = log_command($module, 'git-stash-push', [
+ qw(git stash push -u --quiet --message), $stashName
+ ]) unless pretending(); # probably best not to do anything if pretending()
+
+ #
+ # This might happen if the repo is already in merge conflict state.
+ # We could sledgehammer our way past this by marking everything as resolved using git add . before
+ # stashing, but... that might not always be appreciated by people having to figure out what the
+ # original merge conflicts were afterwards.
+ #
+ if ($status != 0) {
+ log_command($module, 'git-status-after-error', [qw(git status)]);
- $self->_notifyPostBuildMessage(
++ $module->addPostBuildMessage(
+ "b[$module] may have local changes that we couldn't handle, so the module was left alone."
+ );
+ croak_runtime("Unable to stash local changes (if any) for $module, aborting update.");
+ }
- if ($needsStash) {
- info ("\tLocal changes detected (will stash for now and then restore)");
- $status = log_command($module, 'git-stash-save', [
- qw(git stash save --quiet), "kdesrc-build auto-stash at $date",
- ]);
- if ($status != 0) {
- log_command($module, 'git-status-after-error', [qw(git status)]);
- $self->{ipc}->notifyNewPostBuildMessage(
- $module->name(), "b[$module] has local changes that we couldn't handle, so the module was left alone.");
- croak_runtime("Unable to stash local changes for $module, aborting update.");
- }
+ #
+ # next: check if the stash was truly necessary.
+ # compare counts (not just testing if there is *any* stash) because there might have been a
+ # genuine user's stash already prior to kdesrc-build taking over the reins in the repo.
+ #
+ my $newStashCount = $self->countStash();
+ if ($newStashCount != $oldStashCount) {
+ my $message = "b[$module] had local changes that we stashed, ".
+ "you should manually inspect the new stash: b[$stashName]";
+ warning ($message);
- $self->_notifyPostBuildMessage($message);
++ $module->addPostBuildMessage($message);
}
+ # finally, update to remote head
if (!$updateSub->()) {
error ("\tUnable to update the source code for r[b[$module]");
log_command($module, 'git-status-after-error', [qw(git status)]);
diff --cc vim/syntax/kdesrc-buildrc.vim
index d26170a,26606fa..3e6ef72
--- a/vim/syntax/kdesrc-buildrc.vim
+++ b/vim/syntax/kdesrc-buildrc.vim
@@@ -36,17 -36,18 +36,18 @@@ setlocal iskeyword+=
" Keywords
syn keyword ksbrcOption contained skipwhite nextgroup=ksbrcStringValue
- \ binpath branch build-dir checkout-only cmake-options configure-flags
+ \ binpath branch build-dir cmake-options configure-flags
\ custom-build-command cxxflags dest-dir do-not-compile kdedir
\ libpath log-dir make-install-prefix make-options module-base-path
- \ ninja-options
+ \ cmake-generator cmake-toolchain ninja-options
\ override-build-system override-url prefix qtdir repository
\ revision source-dir svn-server tag remove-after-install
\ qmake-options git-user
syn keyword ksbrcGlobalOption contained skipwhite nextgroup=ksbrcStringValue
\ branch-group git-desired-protocol git-repository-base http-proxy
- \ kde-languages niceness debug-level persistent-data-file set-env
+ \ niceness debug-level persistent-data-file set-env
+ \ num-cores num-cores-low-mem
" MUST BE CONSISTENT WITH ABOVE. Used when a module-set option is used in the
" wrong spot to highlight the error.
More information about the kde-doc-english
mailing list