lei_to_mail: write directly to mail_sync.sqlite3

No need to go through the lei/store process when we write
mail_sync.sqlite3.  This ought to reduce ENOBUFS errors (and the
sleep workaround) on RAM-starved systems.

diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index effc572f6aaf45e658b1434537081342e29efe7e..3d0e5b144d3a9c939cc036e525283adc020c8a2c 100644 (file)
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -32,10 +32,13 @@ sub _start_query { # used by "lei q" and "lei up"
        if ($l2m && grep { $opt->{$_} //= \1 } (qw(mail-sync import-remote
                                                        import-before))) {
                $self->_lei_store(1)->write_prepare($self);
+               if ($opt->{'mail-sync'}) {
+                       my $lms = $l2m->{-lms_rw} = $self->lms(1);
+                       $lms->lms_write_prepare->lms_pause; # just create
+               }
        }
-       $l2m and $l2m->{-wq_nr_workers} = $mj // do {
-               $mj = int($nproc * 0.75 + 0.5); # keep some CPU for git
-       };
+       $l2m and $l2m->{-wq_nr_workers} = $mj //
+               int($nproc * 0.75 + 0.5); # keep some CPU for git
 
        # descending docid order is cheapest, MUA controls sorting order
        $self->{mset_opt}->{relevance} //= -2 if $l2m || $opt->{threads};

diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index f1316229bb3205c9395c345de49d4f89f003561b..66049dfeac84c0cb36cd124b331c85a36395b965 100644 (file)
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -265,11 +265,6 @@ sub _lms_rw ($) { # it is important to have eidx processes open before lms
        };
 }
 
-sub set_sync_info {
-       my ($self, $oidhex, $folder, $id) = @_;
-       _lms_rw($self)->set_src(pack('H*', $oidhex), $folder, $id);
-}
-
 sub _remove_if_local { # git->cat_async arg
        my ($bref, $oidhex, $type, $size, $self) = @_;
        $self->{im}->remove($bref) if $bref;
@@ -349,7 +344,7 @@ sub add_eml {
        $smsg->{-eidx_git} = $eidx->git if !$self->{-fake_im};
        my $im_mark = $im->add($eml, undef, $smsg);
        if ($vmd && $vmd->{sync_info}) {
-               set_sync_info($self, $smsg->{blob}, @{$vmd->{sync_info}});
+               _lms_rw($self)->set_src($smsg->oidbin, @{$vmd->{sync_info}});
        }
        unless ($im_mark) { # duplicate blob returns undef
                return unless wantarray;

diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index d33d27aec00646879c89f702b4c3610775f47b09..e7220bf02954f3c7c5382e42f7d0f35e6e7df266 100644 (file)
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -135,8 +135,8 @@ sub git_to_mail { # git->cat_async callback
        my ($bref, $oid, $type, $size, $arg) = @_;
        $type // return; # called by git->async_abort
        my ($write_cb, $smsg) = @$arg;
-       if ($type eq 'missing' && $smsg->{-lms_ro}) {
-               if ($bref = $smsg->{-lms_ro}->local_blob($oid, 1)) {
+       if ($type eq 'missing' && $smsg->{-lms_rw}) {
+               if ($bref = $smsg->{-lms_rw}->local_blob($oid, 1)) {
                        $type = 'blob';
                        $size = length($$bref);
                }
@@ -280,8 +280,8 @@ sub _maildir_write_cb ($$) {
        $dedupe->prepare_dedupe if $dedupe;
        my $dst = $lei->{ovv}->{dst};
        my $lse = $lei->{lse}; # may be undef
-       my $sto = $lei->{opt}->{'mail-sync'} ? $lei->{sto} : undef;
-       my $out = $sto ? 'maildir:'.$lei->abs_path($dst) : undef;
+       my $lms = $self->{-lms_rw};
+       my $out = $lms ? 'maildir:'.$lei->abs_path($dst) : undef;
 
        # Favor cur/ and only write to new/ when augmenting.  This
        # saves MUAs from having to do a mass rename when the initial
@@ -296,7 +296,7 @@ sub _maildir_write_cb ($$) {
                $lse->xsmsg_vmd($smsg) if $lse;
                my $n = _buf2maildir($dst, $bref // \($eml->as_string),
                                        $smsg, $dir);
-               $sto->wq_do('set_sync_info', $smsg->{blob}, $out, $n) if $sto;
+               $lms->set_src($smsg->oidbin, $out, $n) if $lms;
                ++$lei->{-nr_write};
        }
 }
@@ -311,7 +311,7 @@ sub _imap_write_cb ($$) {
        my $folder = $uri->mailbox;
        $uri->uidvalidity($mic->uidvalidity($folder));
        my $lse = $lei->{lse}; # may be undef
-       my $sto = $lei->{opt}->{'mail-sync'} ? $lei->{sto} : undef;
+       my $lms = $self->{-lms_rw};
        sub { # for git_to_mail
                my ($bref, $smsg, $eml) = @_;
                $mic // return $lei->fail; # mic may be undef-ed in last run
@@ -325,9 +325,8 @@ sub _imap_write_cb ($$) {
                        die $err;
                }
                # imap_append returns UID if IMAP server has UIDPLUS extension
-               ($sto && $uid =~ /\A[0-9]+\z/) and
-                       $sto->wq_do('set_sync_info',
-                                       $smsg->{blob}, $$uri, $uid + 0);
+               ($lms && $uid =~ /\A[0-9]+\z/) and
+                       $lms->set_src($smsg->oidbin, $$uri, $uid + 0);
                ++$lei->{-nr_write};
        }
 }
@@ -751,9 +750,7 @@ sub ipc_atfork_child {
        my ($self) = @_;
        my $lei = $self->{lei};
        $lei->_lei_atfork_child;
-       if (my $lse = $lei->{lse}) {
-               $self->{-lms_ro} = $lse->{-lms_ro} //= $lse->lms;
-       }
+       $self->{-lms_rw}->lms_write_prepare if $self->{-lms_rw};
        $lei->{auth}->do_auth_atfork($self) if $lei->{auth};
        $SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
        $self->SUPER::ipc_atfork_child;
@@ -775,7 +772,7 @@ sub poke_dst {
 sub write_mail { # via ->wq_io_do
        my ($self, $smsg, $eml) = @_;
        return $self->{wcb}->(undef, $smsg, $eml) if $eml;
-       $smsg->{-lms_ro} = $self->{-lms_ro};
+       $smsg->{-lms_rw} = $self->{-lms_rw};
        $self->{lei}->{ale}->git->cat_async($smsg->{blob}, \&git_to_mail,
                                [$self->{wcb}, $smsg]);
 }