lei: commit store on interrupted partial imports

[public-inbox.git] / lib / PublicInbox / LeiNoteEvent.pm

diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index c03c53193b032297da2d9abee056c3892a9844ab..db387633e877c9e3b8bbfd3c4458a12b8695188a 100644 (file)
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -7,20 +7,21 @@ package PublicInbox::LeiNoteEvent;
 use strict;
 use v5.10.1;
 use parent qw(PublicInbox::IPC);
+use PublicInbox::DS;
+use Errno qw(ENOENT);
 
-my $flush_timer;
 our $to_flush; # { cfgpath => $lei }
 
-sub flush_lei ($) {
-       my ($lei) = @_;
-       my $lne = delete $lei->{cfg}->{-lei_note_event};
-       $lne->wq_close(1, undef, $lei) if $lne; # runs _lei_wq_eof;
+sub flush_lei ($;$) {
+       my ($lei, $manual) = @_;
+       my $lne = delete $lei->{cfg}->{-lei_note_event} // return;
+       $lne->{lei_sock} = $lei->{sock} if $manual;
+       $lne->wq_close; # runs _lei_wq_eof;
 }
 
 # we batch up writes and flush every 5s (matching Linux default
 # writeback behavior) since MUAs can trigger a storm of inotify events
 sub flush_task { # PublicInbox::DS timer callback
-       undef $flush_timer;
        my $todo = $to_flush // return;
        $to_flush = undef;
        for my $lei (values %$todo) { flush_lei($lei) }
@@ -29,7 +30,7 @@ sub flush_task { # PublicInbox::DS timer callback
 # sets a timer to flush
 sub note_event_arm_done ($) {
        my ($lei) = @_;
-       $flush_timer //= PublicInbox::DS::add_timer(5, \&flush_task);
+       PublicInbox::DS::add_uniq_timer('flush_timer', 5, \&flush_task);
        $to_flush->{$lei->{cfg}->{'-f'}} //= $lei;
 }
 
@@ -37,18 +38,20 @@ sub eml_event ($$$$) {
        my ($self, $eml, $vmd, $state) = @_;
        my $sto = $self->{lei}->{sto};
        if ($state =~ /\Aimport-(?:rw|ro)\z/) {
-               $sto->ipc_do('set_eml', $eml, $vmd);
+               $sto->wq_do('set_eml', $eml, $vmd);
        } elsif ($state =~ /\Aindex-(?:rw|ro)\z/) {
                my $xoids = $self->{lei}->ale->xoids_for($eml);
-               $sto->ipc_do('index_eml_only', $eml, $vmd, $xoids);
+               $sto->wq_do('index_eml_only', $eml, $vmd, $xoids);
        } elsif ($state =~ /\Atag-(?:rw|ro)\z/) {
                my $docids = [];
-               my $c = $self->{lse}->kw_changed($eml, $vmd->{kw}, $docids);
+               my $c = eval {
+                       $self->{lse}->kw_changed($eml, $vmd->{kw}, $docids);
+               } // 1; # too new, assume changed since still to-be-committed.
                if (scalar @$docids) { # already in lei/store
-                       $sto->ipc_do('set_eml_vmd', undef, $vmd, $docids) if $c;
+                       $sto->wq_do('set_eml_vmd', undef, $vmd, $docids) if $c;
                } elsif (my $xoids = $self->{lei}->ale->xoids_for($eml)) {
                        # it's in an external, only set kw, here
-                       $sto->ipc_do('set_xvmd', $xoids, $eml, $vmd);
+                       $sto->wq_do('set_xvmd', $xoids, $eml, $vmd);
                } # else { totally unknown: ignore
        } else {
                warn "unknown state: $state (in $self->{lei}->{cfg}->{'-f'})\n";
@@ -57,8 +60,11 @@ sub eml_event ($$$$) {
 
 sub maildir_event { # via wq_io_do
        my ($self, $fn, $vmd, $state) = @_;
-       my $eml = PublicInbox::InboxWritable::eml_from_path($fn) // return;
-       eml_event($self, $eml, $vmd, $state);
+       if (my $eml = PublicInbox::InboxWritable::eml_from_path($fn)) {
+               eml_event($self, $eml, $vmd, $state);
+       } elsif ($! == ENOENT) {
+               $self->{lms}->clear_src(@{$vmd->{sync_info}});
+       } # else: eml_from_path already warns
 }
 
 sub lei_note_event {
@@ -66,12 +72,12 @@ sub lei_note_event {
        die "BUG: unexpected: @rest" if @rest;
        my $cfg = $lei->_lei_cfg or return; # gone (race)
        my $sto = $lei->_lei_store or return; # gone
-       return flush_lei($lei) if $folder eq 'done'; # special case
+       return flush_lei($lei, 1) if $folder eq 'done'; # special case
        my $lms = $lei->lms or return;
        $lms->lms_write_prepare if $new_cur eq ''; # for ->clear_src below
-       my $err = $lms->arg2folder($lei, [ $folder ]);
-       return if $err->{fail};
-       my $state = $cfg->get_1("watch.$folder", 'state') // 'tag-rw';
+       $lei->{opt}->{quiet} = 1;
+       $lms->arg2folder($lei, [ $folder ]);
+       my $state = $cfg->get_1("watch.$folder.state") // 'tag-rw';
        return if $state eq 'pause';
        return $lms->clear_src($folder, \$bn) if $new_cur eq '';
        $lms->lms_pause;
@@ -81,8 +87,9 @@ sub lei_note_event {
        my $self = $cfg->{-lei_note_event} //= do {
                my $wq = bless { lms => $lms }, __PACKAGE__;
                # MUAs such as mutt can trigger massive rename() storms so
-               # use all CPU power available:
+               # use some CPU, but don't overwhelm slower storage, either
                my $jobs = $wq->detect_nproc // 1;
+               $jobs = 4 if $jobs > 4; # same default as V2Writable
                my ($op_c, $ops) = $lei->workers_start($wq, $jobs);
                $lei->wait_wq_events($op_c, $ops);
                note_event_arm_done($lei);
@@ -94,7 +101,7 @@ sub lei_note_event {
                return if index($fl, 'T') >= 0;
                my $kw = PublicInbox::MdirReader::flags2kw($fl);
                my $vmd = { kw => $kw, sync_info => [ $folder, \$bn ] };
-               $self->wq_io_do('maildir_event', [], $fn, $vmd, $state);
+               $self->wq_do('maildir_event', $fn, $vmd, $state);
        } # else: TODO: imap
 }
 
@@ -106,17 +113,8 @@ sub ipc_atfork_child {
        $self->SUPER::ipc_atfork_child;
 }
 
-sub lne_done_wait {
-       my ($arg, $pid) = @_;
-       my ($self, $lei) = @$arg;
-       $lei->can('wq_done_wait')->($arg, $pid);
-}
-
 sub _lei_wq_eof { # EOF callback for main lei daemon
-       my ($lei) = @_;
-       my $lne = delete $lei->{lne} or return $lei->fail;
-       $lei->sto_done_request;
-       $lne->wq_wait_old(\&lne_done_wait, $lei);
+       $_[0]->wq_eof('lne');
 }
 
 1;