X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FLeiStore.pm;h=08add8f5a05496ab234f5cfad9737903de0c009a;hb=47d4e53734820b4e;hp=fcc9224d4644d7c2c6d97a96a99128155539c329;hpb=63e2751ace2dd35a661620a9ad30d6c92948a009;p=public-inbox.git diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm index fcc9224d..08add8f5 100644 --- a/lib/PublicInbox/LeiStore.pm +++ b/lib/PublicInbox/LeiStore.pm @@ -25,9 +25,13 @@ use PublicInbox::MID qw(mids); use PublicInbox::LeiSearch; use PublicInbox::MDA; use PublicInbox::Spawn qw(spawn); -use List::Util qw(max); +use PublicInbox::MdirReader; +use PublicInbox::LeiToMail; use File::Temp (); use POSIX (); +use IO::Handle (); # ->autoflush +use Sys::Syslog qw(syslog openlog); +use Errno qw(EEXIST ENOENT); sub new { my (undef, $dir, $opt) = @_; @@ -45,19 +49,6 @@ sub rotate_bytes { $_[0]->{rotate_bytes} // ((1024 * 1024 * 1024) / $_[0]->packing_factor) } -sub git_pfx { "$_[0]->{priv_eidx}->{topdir}/local" }; - -sub git_epoch_max { - my ($self) = @_; - if (opendir(my $dh, $self->git_pfx)) { - max(map { - substr($_, 0, -4) + 0; # drop ".git" suffix - } grep(/\A[0-9]+\.git\z/, readdir($dh))) // 0; - } else { - $!{ENOENT} ? 0 : die("opendir ${\$self->git_pfx}: $!\n"); - } -} - sub git_ident ($) { my ($git) = @_; my $rdr = {}; @@ -86,19 +77,16 @@ sub importer { $im->done; undef $im; $self->checkpoint; - $max = $self->git_epoch_max + 1; + $max = $self->{priv_eidx}->{mg}->git_epochs + 1; } - my $pfx = $self->git_pfx; - $max //= $self->git_epoch_max; + my (undef, $tl) = eidx_init($self); # acquire lock + $max //= $self->{priv_eidx}->{mg}->git_epochs; while (1) { - my $latest = "$pfx/$max.git"; - my $old = -e $latest; - PublicInbox::Import::init_bare($latest); + my $latest = $self->{priv_eidx}->{mg}->add_epoch($max); my $git = PublicInbox::Git->new($latest); - if (!$old) { - $git->qx(qw(config core.sharedRepository 0600)); - $self->done; # force eidx_init on next round - } + $self->done; # unlock + # re-acquire lock, update alternates for new epoch + (undef, $tl) = eidx_init($self); my $packed_bytes = $git->packed_bytes; my $unpacked_bytes = $packed_bytes / $self->packing_factor; if ($unpacked_bytes >= $self->rotate_bytes) { @@ -130,7 +118,7 @@ sub eidx_init { my $tl = wantarray && $self->{-err_wr} ? PublicInbox::OnDestroy->new($$, \&_tail_err, $self) : undef; - $eidx->idx_init({-private => 1}); + $eidx->idx_init({-private => 1}); # acquires lock wantarray ? ($eidx, $tl) : $eidx; } @@ -148,7 +136,8 @@ sub _docids_for ($$) { my $oid = $cur->{blob}; my $docid = $cur->{num}; my $bref = $im ? $im->cat_blob($oid) : undef; - $bref //= $eidx->git->cat_file($oid) // do { + $bref //= $eidx->git->cat_file($oid) // + _lms_rw($self)->local_blob($oid, 1) // do { warn "W: $oid (#$docid) <$mid> not found\n"; next; }; @@ -160,12 +149,88 @@ sub _docids_for ($$) { sort { $a <=> $b } values %docids; } +# n.b. similar to LeiExportKw->export_kw_md, but this is for a single eml +sub export1_kw_md ($$$$$) { + my ($self, $mdir, $bn, $oidbin, $vmdish) = @_; # vmd/vmd_mod + my $orig = $bn; + my (@try, $unkn, $kw); + if ($bn =~ s/:2,([a-zA-Z]*)\z//) { + ($kw, $unkn) = PublicInbox::MdirReader::flags2kw($1); + if (my $set = $vmdish->{kw}) { + $kw = $set; + } elsif (my $add = $vmdish->{'+kw'}) { + @$kw{@$add} = (); + } elsif (my $del = $vmdish->{-kw}) { + delete @$kw{@$del}; + } # else no changes... + @try = qw(cur new); + } else { # no keywords, yet, could be in new/ + @try = qw(new cur); + $unkn = []; + if (my $set = $vmdish->{kw}) { + $kw = $set; + } elsif (my $add = $vmdish->{'+kw'}) { + @$kw{@$add} = (); # auto-vivify + } else { # ignore $vmdish->{-kw} + $kw = []; + } + } + $kw = [ keys %$kw ] if ref($kw) eq 'HASH'; + $bn .= ':2,'. PublicInbox::LeiToMail::kw2suffix($kw, @$unkn); + return if $orig eq $bn; # no change + + # we use link(2) + unlink(2) since rename(2) may + # inadvertently clobber if the "uniquefilename" part wasn't + # actually unique. + my $dst = "$mdir/cur/$bn"; + for my $d (@try) { + my $src = "$mdir/$d/$orig"; + if (link($src, $dst)) { + if (!unlink($src) and $! != ENOENT) { + syslog('warning', "unlink($src): $!"); + } + # TODO: verify oidbin? + $self->{lms}->mv_src("maildir:$mdir", + $oidbin, \$orig, $bn); + return; + } elsif ($! == EEXIST) { # lost race with "lei export-kw"? + return; + } elsif ($! != ENOENT) { + syslog('warning', "link($src -> $dst): $!"); + } + } + for (@try) { return if -e "$mdir/$_/$orig" }; + $self->{lms}->clear_src("maildir:$mdir", \$orig); +} + +sub sto_export_kw ($$$) { + my ($self, $docid, $vmdish) = @_; # vmdish (vmd or vmd_mod) + my ($eidx, $tl) = eidx_init($self); + my $lms = _lms_rw($self) // return; + my $xr3 = $eidx->{oidx}->get_xref3($docid, 1); + for my $row (@$xr3) { + my (undef, undef, $oidbin) = @$row; + my $locs = $lms->locations_for($oidbin) // next; + while (my ($loc, $ids) = each %$locs) { + if ($loc =~ s!\Amaildir:!!i) { + for my $id (@$ids) { + export1_kw_md($self, $loc, $id, + $oidbin, $vmdish); + } + } + # TODO: IMAP + } + } +} + +# vmd = { kw => [ qw(seen ...) ], L => [ qw(inbox ...) ] } sub set_eml_vmd { my ($self, $eml, $vmd, $docids) = @_; my ($eidx, $tl) = eidx_init($self); $docids //= [ _docids_for($self, $eml) ]; for my $docid (@$docids) { $eidx->idx_shard($docid)->ipc_do('set_vmd', $docid, $vmd); + sto_export_kw($self, $docid, $vmd); } $docids; } @@ -180,7 +245,7 @@ sub add_eml_vmd { \@docids; } -sub remove_eml_vmd { +sub remove_eml_vmd { # remove just the VMD my ($self, $eml, $vmd) = @_; my ($eidx, $tl) = eidx_init($self); my @docids = _docids_for($self, $eml); @@ -190,26 +255,90 @@ sub remove_eml_vmd { \@docids; } -sub set_sync_info ($$$) { - my ($self, $oidhex, $sync_info) = @_; - ($self->{lms} //= do { +sub _lms_rw ($) { # it is important to have eidx processes open before lms + my ($self) = @_; + my ($eidx, $tl) = eidx_init($self); + $self->{lms} //= do { require PublicInbox::LeiMailSync; my $f = "$self->{priv_eidx}->{topdir}/mail_sync.sqlite3"; my $lms = PublicInbox::LeiMailSync->new($f); - $lms->lms_begin; + $lms->lms_write_prepare; $lms; - })->set_src($oidhex, @$sync_info); + }; +} + +sub set_sync_info { + my ($self, $oidhex, $folder, $id) = @_; + _lms_rw($self)->set_src(pack('H*', $oidhex), $folder, $id); +} + +sub _remove_if_local { # git->cat_async arg + my ($bref, $oidhex, $type, $size, $self) = @_; + $self->{im}->remove($bref) if $bref; +} + +sub remove_docids ($;@) { + my ($self, @docids) = @_; + my $eidx = eidx_init($self); + for my $docid (@docids) { + $eidx->idx_shard($docid)->ipc_do('xdb_remove', $docid); + $eidx->{oidx}->delete_by_num($docid); + $eidx->{oidx}->{dbh}->do(<importer; # may create new epoch + my ($eidx, $tl) = eidx_init($self); + my $oidx = $eidx->{oidx}; + my @docids = _docids_for($self, $eml); + my $git = $eidx->git; + for my $docid (@docids) { + my $xr3 = $oidx->get_xref3($docid, 1); + for my $row (@$xr3) { + my (undef, undef, $oidbin) = @$row; + my $oidhex = unpack('H*', $oidbin); + $git->cat_async($oidhex, \&_remove_if_local, $self); + } + } + $git->cat_async_wait; + remove_docids($self, @docids); + \@docids; +} + +sub oid2docid ($$) { + my ($self, $oid) = @_; + my $eidx = eidx_init($self); + my ($docid, @cull) = $eidx->{oidx}->blob_exists($oid); + if (@cull) { # fixup old bugs... + warn <ipc_do('add_vmd', $docid, $vmd); + sto_export_kw($self, $docid, $vmd); } sub add_eml { my ($self, $eml, $vmd, $xoids) = @_; - my $im = $self->importer; # may create new epoch - my ($eidx, $tl) = eidx_init($self); # updates/writes alternates file + my $im = $self->{-fake_im} // $self->importer; # may create new epoch + my ($eidx, $tl) = eidx_init($self); my $oidx = $eidx->{oidx}; # PublicInbox::Import::add checks this my $smsg = bless { -oidx => $oidx }, 'PublicInbox::Smsg'; + $smsg->{-eidx_git} = $eidx->git if !$self->{-fake_im}; my $im_mark = $im->add($eml, undef, $smsg); if ($vmd && $vmd->{sync_info}) { - set_sync_info($self, $smsg->{blob}, $vmd->{sync_info}); + set_sync_info($self, $smsg->{blob}, @{$vmd->{sync_info}}); } $im_mark or return; # duplicate blob returns undef @@ -220,13 +349,13 @@ sub add_eml { if (scalar keys %$xoids) { my %docids = map { $_ => 1 } @$vivify_xvmd; for my $oid (keys %$xoids) { - my @id = $oidx->blob_exists($oid); - @docids{@id} = @id; + my $docid = oid2docid($self, $oid); + $docids{$docid} = $docid if defined($docid); } @$vivify_xvmd = sort { $a <=> $b } keys(%docids); } } - if (@$vivify_xvmd) { + if (@$vivify_xvmd) { # docids list $xoids //= {}; $xoids->{$smsg->{blob}} = 1; for my $docid (@$vivify_xvmd) { @@ -243,7 +372,7 @@ sub add_eml { for my $oid (keys %$xoids) { $oidx->add_xref3($docid, -1, $oid, '.'); } - $idx->ipc_do('add_vmd', $docid, $vmd) if $vmd; + _add_vmd($self, $idx, $docid, $vmd) if $vmd; } $vivify_xvmd; } elsif (my @docids = _docids_for($self, $eml)) { @@ -253,16 +382,17 @@ sub add_eml { $oidx->add_xref3($docid, -1, $smsg->{blob}, '.'); # add_eidx_info for List-Id $idx->ipc_do('add_eidx_info', $docid, '.', $eml); - $idx->ipc_do('add_vmd', $docid, $vmd) if $vmd; + _add_vmd($self, $idx, $docid, $vmd) if $vmd; } \@docids; } else { # totally new message + delete $smsg->{-oidx}; # for IPC-friendliness $smsg->{num} = $oidx->adj_counter('eidx_docid', '+'); $oidx->add_overview($eml, $smsg); $oidx->add_xref3($smsg->{num}, -1, $smsg->{blob}, '.'); my $idx = $eidx->idx_shard($smsg->{num}); $idx->index_eml($eml, $smsg); - $idx->ipc_do('add_vmd', $smsg->{num}, $vmd) if $vmd; + _add_vmd($self, $idx, $smsg->{num}, $vmd) if $vmd; $smsg; } } @@ -273,6 +403,14 @@ sub set_eml { set_eml_vmd($self, $eml, $vmd); } +sub index_eml_only { + my ($self, $eml, $vmd, $xoids) = @_; + require PublicInbox::FakeImport; + local $self->{-fake_im} = PublicInbox::FakeImport->new; + set_eml($self, $eml, $vmd, $xoids); +} + +# store {kw} / {L} info for a message which is only in an external sub _external_only ($$$) { my ($self, $xoids, $eml) = @_; my $eidx = $self->{priv_eidx}; @@ -301,15 +439,12 @@ sub update_xvmd { my $oidx = $eidx->{oidx}; my %seen; for my $oid (keys %$xoids) { - my @docids = $oidx->blob_exists($oid) or next; - scalar(@docids) > 1 and - warn "W: $oid indexed as multiple docids: @docids\n"; - for my $docid (@docids) { - next if $seen{$docid}++; - my $idx = $eidx->idx_shard($docid); - $idx->ipc_do('update_vmd', $docid, $vmd_mod); - } + my $docid = oid2docid($self, $oid) // next; delete $xoids->{$oid}; + next if $seen{$docid}++; + my $idx = $eidx->idx_shard($docid); + $idx->ipc_do('update_vmd', $docid, $vmd_mod); + sto_export_kw($self, $docid, $vmd_mod); } return unless scalar(keys(%$xoids)); @@ -322,12 +457,14 @@ sub update_xvmd { } my $idx = $eidx->idx_shard($docid); $idx->ipc_do('update_vmd', $docid, $vmd_mod); + sto_export_kw($self, $docid, $vmd_mod); } return; } # totally unseen my ($smsg, $idx) = _external_only($self, $xoids, $eml); $idx->ipc_do('update_vmd', $smsg->{num}, $vmd_mod); + sto_export_kw($self, $smsg->{num}, $vmd_mod); } # set or update keywords for external message, called via ipc_do @@ -340,15 +477,12 @@ sub set_xvmd { # see if we can just update existing docs for my $oid (keys %$xoids) { - my @docids = $oidx->blob_exists($oid) or next; - scalar(@docids) > 1 and - warn "W: $oid indexed as multiple docids: @docids\n"; - for my $docid (@docids) { - next if $seen{$docid}++; - my $idx = $eidx->idx_shard($docid); - $idx->ipc_do('set_vmd', $docid, $vmd); - } + my $docid = oid2docid($self, $oid) // next; delete $xoids->{$oid}; # all done with this oid + next if $seen{$docid}++; + my $idx = $eidx->idx_shard($docid); + $idx->ipc_do('set_vmd', $docid, $vmd); + sto_export_kw($self, $docid, $vmd); } return unless scalar(keys(%$xoids)); @@ -359,6 +493,7 @@ sub set_xvmd { # totally unseen: my ($smsg, $idx) = _external_only($self, $xoids, $eml); $idx->ipc_do('add_vmd', $smsg->{num}, $vmd); + sto_export_kw($self, $smsg->{num}, $vmd); } sub checkpoint { @@ -366,6 +501,7 @@ sub checkpoint { if (my $im = $self->{im}) { $wait ? $im->barrier : $im->checkpoint; } + delete $self->{lms}; $self->{priv_eidx}->checkpoint($wait); } @@ -385,7 +521,7 @@ sub xchg_stderr { } sub done { - my ($self) = @_; + my ($self, $sock_ref) = @_; my $err = ''; if (my $im = delete($self->{im})) { eval { $im->done }; @@ -394,12 +530,14 @@ sub done { warn $err; } } - if (my $lms = delete $self->{lms}) { - $lms->lms_commit; - } + delete $self->{lms}; $self->{priv_eidx}->done; # V2Writable::done xchg_stderr($self); die $err if $err; + + # notify clients ->done has been issued + defined($sock_ref) and + $self->{s2d_op_p}->pkt_do('sto_done_complete', $sock_ref); } sub ipc_atfork_child { @@ -407,33 +545,39 @@ sub ipc_atfork_child { my $lei = $self->{lei}; $lei->_lei_atfork_child(1) if $lei; xchg_stderr($self); - if (my $err = delete($self->{err_pipe})) { - close $err->[0]; - $self->{-err_wr} = $err->[1]; + if (my $to_close = delete($self->{to_close})) { + close($_) for @$to_close; } + openlog('lei/store', 'pid,nowait,nofatal,ndelay', 'user'); $self->SUPER::ipc_atfork_child; } sub write_prepare { my ($self, $lei) = @_; + $lei // die 'BUG: $lei not passed'; unless ($self->{-ipc_req}) { - my $d = $lei->store_path; - $self->ipc_lock_init("$d/ipc.lock"); - substr($d, -length('/lei/store'), 10, ''); - my $err_pipe; - unless ($lei->{oneshot}) { - pipe(my ($r, $w)) or die "pipe: $!"; - $err_pipe = [ $r, $w ]; - } + # s2d => store-to-daemon messages + require PublicInbox::PktOp; + my ($s2d_op_c, $s2d_op_p) = PublicInbox::PktOp->pair; + my $dir = $lei->store_path; + $self->ipc_lock_init("$dir/ipc.lock"); + substr($dir, -length('/lei/store'), 10, ''); + pipe(my ($r, $w)) or die "pipe: $!"; + $w->autoflush(1); # Mail we import into lei are private, so headers filtered out # by -mda for public mail are not appropriate local @PublicInbox::MDA::BAD_HEADERS = (); - $self->ipc_worker_spawn("lei/store $d", $lei->oldset, - { lei => $lei, err_pipe => $err_pipe }); - if ($err_pipe) { - require PublicInbox::LeiStoreErr; - PublicInbox::LeiStoreErr->new($err_pipe->[0], $lei); - } + $self->ipc_worker_spawn("lei/store $dir", $lei->oldset, { + lei => $lei, + -err_wr => $w, + to_close => [ $r, $s2d_op_c->{sock} ], + s2d_op_p => $s2d_op_p, + }); + require PublicInbox::LeiStoreErr; + PublicInbox::LeiStoreErr->new($r, $lei); + $s2d_op_c->{ops} = { + sto_done_complete => [ $lei->can('sto_done_complete') ] + }; } $lei->{sto} = $self; }