lei: lock worker counts

[public-inbox.git] / lib / PublicInbox / LEI.pm

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 784e679d564a9eb8d8bab0d8ec2641248fd9747b..41e761f8aabf939e81af2afb0b82bf7058b83e5e 100644 (file)
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -24,6 +24,8 @@ use PublicInbox::DS qw(now dwaitpid);
 use PublicInbox::Spawn qw(spawn popen_rd);
 use PublicInbox::Lock;
 use PublicInbox::Eml;
+use PublicInbox::Import;
+use PublicInbox::ContentHash qw(git_sha);
 use Time::HiRes qw(stat); # ctime comparisons for config cache
 use File::Path qw(mkpath);
 use File::Spec;
@@ -256,12 +258,12 @@ our %CMD = ( # sorted in order of importance/use:
         @c_opt ],
 'import' => [ 'LOCATION...|--stdin',
        'one-time import/update from URL or filesystem',
-       qw(stdin| offset=i recursive|r exclude=s include|I=s jobs=s new-only
+       qw(stdin| offset=i recursive|r exclude=s include|I=s new-only
        lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
        @net_opt, @c_opt ],
 'forget-mail-sync' => [ 'LOCATION...',
        'forget sync information for a mail folder', @c_opt ],
-'prune-mail-sync' => [ 'LOCATION...|--all',
+'refresh-mail-sync' => [ 'LOCATION...|--all',
        'prune dangling sync data for a mail folder', 'all:s', @c_opt ],
 'export-kw' => [ 'LOCATION...|--all',
        'one-time export of keywords of sync sources',
@@ -614,6 +616,7 @@ sub pkt_ops {
        $ops->{x_it} = [ \&x_it, $lei ];
        $ops->{child_error} = [ \&child_error, $lei ];
        $ops->{incr} = [ \&incr, $lei ];
+       $ops->{sto_done_request} = [ \&sto_done_request, $lei, $lei->{sock} ];
        $ops;
 }
 
@@ -624,6 +627,7 @@ sub workers_start {
        my $end = $lei->pkt_op_pair;
        my $ident = $wq->{-wq_ident} // "lei-$lei->{cmd} worker";
        $flds->{lei} = $lei;
+       $wq->{-wq_nr_workers} //= $jobs; # lock, no incrementing
        $wq->wq_workers_start($ident, $jobs, $lei->oldset, $flds);
        delete $lei->{pkt_op_p};
        my $op_c = delete $lei->{pkt_op_c};
@@ -769,6 +773,8 @@ sub lazy_cb ($$$) {
 
 sub dispatch {
        my ($self, $cmd, @argv) = @_;
+       fchdir($self) or return;
+       local %ENV = %{$self->{env}};
        local $current_lei = $self; # for __WARN__
        $self->{2}->autoflush(1); # keep stdout buffered until x_it|DESTROY
        return _help($self, 'no command given') unless defined($cmd);
@@ -1104,14 +1110,9 @@ sub accept_dispatch { # Listener {post_accept} callback
        my ($argc, @argv) = split(/\0/, $buf, -1);
        undef $buf;
        my %env = map { split(/=/, $_, 2) } splice(@argv, $argc);
-       if (chdir($self->{3})) {
-               local %ENV = %env;
-               $self->{env} = \%env;
-               eval { dispatch($self, @argv) };
-               send($sock, $@, MSG_EOR) if $@;
-       } else {
-               send($sock, "fchdir: $!", MSG_EOR); # implicit close
-       }
+       $self->{env} = \%env;
+       eval { dispatch($self, @argv) };
+       send($sock, $@, MSG_EOR) if $@;
 }
 
 sub dclose {
@@ -1181,7 +1182,6 @@ sub cfg2lei ($) {
        open($lei->{1}, '>>&', \*STDOUT) or die "dup 1: $!";
        open($lei->{2}, '>>&', \*STDERR) or die "dup 2: $!";
        open($lei->{3}, '/') or die "open /: $!";
-       chdir($lei->{3}) or die "chdir /': $!";
        my ($x, $y);
        socketpair($x, $y, AF_UNIX, SOCK_SEQPACKET, 0) or die "socketpair: $!";
        $lei->{sock} = $x;
@@ -1199,7 +1199,6 @@ sub dir_idle_handler ($) { # PublicInbox::DirIdle callback
                for my $f (keys %{$MDIR2CFGPATH->{$mdir} // {}}) {
                        my $cfg = $PATH2CFG{$f} // next;
                        eval {
-                               local %ENV = %{$cfg->{-env}};
                                my $lei = cfg2lei($cfg);
                                $lei->dispatch('note-event',
                                                "maildir:$mdir", $nc, $bn, $fn);
@@ -1347,7 +1346,8 @@ sub lazy_start {
        open STDERR, '>&STDIN' or die "redirect stderr failed: $!";
        open STDOUT, '>&STDIN' or die "redirect stdout failed: $!";
        # $daemon pipe to `lei' closed, main loop begins:
-       PublicInbox::DS->EventLoop;
+       eval { PublicInbox::DS->EventLoop };
+       warn "event loop error: $@\n" if $@;
        dump_and_clear_log();
        exit($exit_code // 0);
 }
@@ -1483,9 +1483,11 @@ sub refresh_watches {
        }
 }
 
-sub git_blob_id {
-       my ($lei, $eml) = @_;
-       ($lei->{sto} // _lei_store($lei, 1))->git_blob_id($eml);
+# TODO: support SHA-256
+sub git_oid {
+       my $eml = $_[-1];
+       $eml->header_set($_) for @PublicInbox::Import::UNWANTED_HEADERS;
+       git_sha(1, $eml);
 }
 
 sub lms { # read-only LeiMailSync
@@ -1499,12 +1501,15 @@ sub lms { # read-only LeiMailSync
 
 sub sto_done_request { # only call this from lei-daemon process (not workers)
        my ($lei, $sock) = @_;
-       if ($sock //= $lei->{sock}) {
-               $LIVE_SOCK{"$sock"} = $sock;
-               $lei->{sto}->ipc_do('done', "$sock"); # issue, async wait
-       } else { # forcibly wait
-               my $wait = $lei->{sto}->ipc_do('done');
-       }
+       eval {
+               if ($sock //= $lei->{sock}) { # issue, async wait
+                       $LIVE_SOCK{"$sock"} = $sock;
+                       $lei->{sto}->ipc_do('done', "$sock");
+               } else { # forcibly wait
+                       my $wait = $lei->{sto}->ipc_do('done');
+               }
+       };
+       $lei->err($@) if $@;
 }
 
 sub sto_done_complete { # called in lei-daemon when LeiStore->done is complete