lei_mirror: retrieve description text asynchronously, too

We can easily parallelize this, so do it.

diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index 6d72f15d52c72a0e9786e2bb3bd1da35a37b7a12..0696c2d996d1f08764242dee6d75e6f0d416b671 100644 (file)
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -122,13 +122,13 @@ sub _get_txt_start { # non-fatal
        spawn($cmd, undef, $opt);
 }
 
-sub _get_txt_done {
+sub _get_txt_done { # returns true on error (non-fatal), undef on success
        my ($self) = @_;
        my ($ft, $cmd, $uri, $file, $mode) = @{delete $self->{-get_txt}};
        my $cerr = $?;
        $? = 0;
-       return "$uri missing" if ($cerr >> 8) == 22;
-       return "# @$cmd failed (non-fatal)" if $cerr;
+       return warn("$uri missing\n") if ($cerr >> 8) == 22;
+       return warn("# @$cmd failed (non-fatal)\n") if $cerr;
        my $dst = $self->{cur_dst} // $self->{dst};
        ft_rename($ft, "$dst/$file", $mode);
        undef; # success
@@ -142,8 +142,7 @@ sub _try_config_start {
 
 sub _try_config_done {
        my ($self) = @_;
-       my $err = _get_txt_done($self);
-       return warn($err, "\n") if $err;
+       _get_txt_done($self) and return;
        my $dst = $self->{cur_dst} // $self->{dst};
        my $f = "$dst/inbox.config.example";
        my $cfg = PublicInbox::Config->git_config_dump($f, $self->{lei}->{2});
@@ -155,11 +154,6 @@ sub _try_config_done {
        }
 }
 
-sub _get_txt { # non-fatal temporary compat function
-       waitpid(_get_txt_start(@_), 0) > 0 or die "waitpid: $!";
-       _get_txt_done($_[0]);
-}
-
 sub set_description ($) {
        my ($self) = @_;
        my $dst = $self->{cur_dst} // $self->{dst};
@@ -180,8 +174,6 @@ sub set_description ($) {
 sub index_cloned_inbox {
        my ($self, $iv) = @_;
        my $lei = $self->{lei};
-       my $err = _get_txt($self, qw(description description), 0666);
-       warn($err, "\n") if $err; # non fatal
        eval { set_description($self) };
        warn $@ if $@;
 
@@ -240,6 +232,9 @@ sub clone_v1 {
        # wait for `git clone' to mkdir $dst (TODO: inotify/kevent?)
        select(undef, undef, undef, 0.011) until -d $dst;
        $LIVE{_try_config_start($self)} = [ \&_try_config_done, $self, $fini ];
+       reap_live() while keys(%LIVE) >= $jobs;
+       $LIVE{_get_txt_start($self, qw(description description), 0666)} =
+                       [ \&_get_txt_done, $self, $fini ];
        reap_live() until ($nohang || !keys(%LIVE));
 }
 
@@ -383,10 +378,13 @@ failed to extract epoch number from $src
        }
        my $lk = bless { lock_path => "$dst/inbox.lock" }, 'PublicInbox::Lock';
        my $fini = PublicInbox::OnDestroy->new($$, \&v2_done, $task);
+       my $jobs = $self->{lei}->{opt}->{jobs} // 1;
        $LIVE{_try_config_start($task)} = [ \&_try_config_done, $task, $fini ];
+       reap_live() while keys(%LIVE) >= $jobs;
+       $LIVE{_get_txt_start($self, qw(description description), 0666)} =
+                               [ \&_get_txt_done, $self, $fini ];
        $task->{-locked} = $lk->lock_for_scope($$);
        my @cmd = clone_cmd($lei, my $opt = {});
-       my $jobs = $self->{lei}->{opt}->{jobs} // 1;
        do {
                reap_live() while keys(%LIVE) >= $jobs;
                while (keys(%LIVE) < $jobs && @src_edst &&