X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FLeiStore.pm;h=f1316229bb3205c9395c345de49d4f89f003561b;hb=4cd7a78f3b8c03670e2d77675229472506eee1eb;hp=a91b30f749520b9ad2eab309d00e168d79fba863;hpb=4166c482b710acea785063de2c69a8a370c6d373;p=public-inbox.git diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm index a91b30f7..f1316229 100644 --- a/lib/PublicInbox/LeiStore.pm +++ b/lib/PublicInbox/LeiStore.pm @@ -20,19 +20,19 @@ use PublicInbox::Eml; use PublicInbox::Import; use PublicInbox::InboxWritable qw(eml_from_path); use PublicInbox::V2Writable; -use PublicInbox::ContentHash qw(content_hash git_sha); +use PublicInbox::ContentHash qw(content_hash); use PublicInbox::MID qw(mids); use PublicInbox::LeiSearch; use PublicInbox::MDA; use PublicInbox::Spawn qw(spawn); use PublicInbox::MdirReader; use PublicInbox::LeiToMail; -use List::Util qw(max); use File::Temp (); use POSIX (); use IO::Handle (); # ->autoflush use Sys::Syslog qw(syslog openlog); use Errno qw(EEXIST ENOENT); +use PublicInbox::Syscall qw(rename_noreplace); sub new { my (undef, $dir, $opt) = @_; @@ -50,19 +50,6 @@ sub rotate_bytes { $_[0]->{rotate_bytes} // ((1024 * 1024 * 1024) / $_[0]->packing_factor) } -sub git_pfx { "$_[0]->{priv_eidx}->{topdir}/local" }; - -sub git_epoch_max { - my ($self) = @_; - if (opendir(my $dh, $self->git_pfx)) { - max(map { - substr($_, 0, -4) + 0; # drop ".git" suffix - } grep(/\A[0-9]+\.git\z/, readdir($dh))) // 0; - } else { - $!{ENOENT} ? 0 : die("opendir ${\$self->git_pfx}: $!\n"); - } -} - sub git_ident ($) { my ($git) = @_; my $rdr = {}; @@ -91,22 +78,16 @@ sub importer { $im->done; undef $im; $self->checkpoint; - $max = $self->git_epoch_max + 1; + $max = $self->{priv_eidx}->{mg}->git_epochs + 1; } my (undef, $tl) = eidx_init($self); # acquire lock - my $pfx = $self->git_pfx; - $max //= $self->git_epoch_max; + $max //= $self->{priv_eidx}->{mg}->git_epochs; while (1) { - my $latest = "$pfx/$max.git"; - my $old = -e $latest; - PublicInbox::Import::init_bare($latest); + my $latest = $self->{priv_eidx}->{mg}->add_epoch($max); my $git = PublicInbox::Git->new($latest); - if (!$old) { - $git->qx(qw(config core.sharedRepository 0600)); - $self->done; # unlock - # re-acquire lock, update alternates for new epoch - (undef, $tl) = eidx_init($self); - } + $self->done; # unlock + # re-acquire lock, update alternates for new epoch + (undef, $tl) = eidx_init($self); my $packed_bytes = $git->packed_bytes; my $unpacked_bytes = $packed_bytes / $self->packing_factor; if ($unpacked_bytes >= $self->rotate_bytes) { @@ -205,26 +186,19 @@ sub export1_kw_md ($$$$$) { my $dst = "$mdir/cur/$bn"; for my $d (@try) { my $src = "$mdir/$d/$orig"; - if (link($src, $dst)) { - if (!unlink($src) and $! != ENOENT) { - syslog('warning', "unlink($src): $!"); - } + if (rename_noreplace($src, $dst)) { # TODO: verify oidbin? - lms_mv_src($self, "maildir:$mdir", + $self->{lms}->mv_src("maildir:$mdir", $oidbin, \$orig, $bn); return; } elsif ($! == EEXIST) { # lost race with "lei export-kw"? return; - } elsif ($! == ENOENT) { - syslog('warning', "link($src -> $dst): $!") - } # else loop @try + } elsif ($! != ENOENT) { + syslog('warning', "rename_noreplace($src -> $dst): $!"); + } } - my $e = $!; - my $src = "$mdir/{".join(',', @try)."}/$orig"; - my $oidhex = unpack('H*', $oidbin); - syslog('warning', "link($src -> $dst) ($oidhex): $e"); for (@try) { return if -e "$mdir/$_/$orig" }; - lms_clear_src($self, "maildir:$mdir", \$orig); + $self->{lms}->clear_src("maildir:$mdir", \$orig); } sub sto_export_kw ($$$) { @@ -279,7 +253,7 @@ sub remove_eml_vmd { # remove just the VMD \@docids; } -sub _lms_rw ($) { +sub _lms_rw ($) { # it is important to have eidx processes open before lms my ($self) = @_; my ($eidx, $tl) = eidx_init($self); $self->{lms} //= do { @@ -291,27 +265,6 @@ sub _lms_rw ($) { }; } -sub lms_clear_src { - my ($self, $folder, $id) = @_; - _lms_rw($self)->clear_src($folder, $id); -} - -sub lms_mv_src { - my ($self, $folder, $oidbin, $id, $newbn) = @_; - _lms_rw($self)->mv_src($folder, $oidbin, $id, $newbn); -} - -sub lms_forget_folders { - my ($self, @folders) = @_; - my $lms = _lms_rw($self); - for my $f (@folders) { $lms->forget_folder($f) } -} - -sub lms_rename_folder { - my ($self, $old, $new) = @_; - _lms_rw($self)->rename_folder($old, $new); -} - sub set_sync_info { my ($self, $oidhex, $folder, $id) = @_; _lms_rw($self)->set_src(pack('H*', $oidhex), $folder, $id); @@ -326,8 +279,7 @@ sub remove_docids ($;@) { my ($self, @docids) = @_; my $eidx = eidx_init($self); for my $docid (@docids) { - $eidx->idx_shard($docid)->ipc_do('xdb_remove', $docid); - $eidx->{oidx}->delete_by_num($docid); + $eidx->remove_doc($docid); $eidx->{oidx}->{dbh}->do(<cat_async($oidhex, \&_remove_if_local, $self); } } - $git->cat_async_wait; + $git->async_wait_all; remove_docids($self, @docids); \@docids; } @@ -374,6 +326,20 @@ sub _add_vmd ($$$$) { sto_export_kw($self, $docid, $vmd); } +sub _docids_and_maybe_kw ($$) { + my ($self, $docids) = @_; + return $docids unless wantarray; + my $kw = {}; + for my $num (@$docids) { # likely only 1, unless ContentHash changes + # can't use ->search->msg_keywords on uncommitted docs + my $idx = $self->{priv_eidx}->idx_shard($num); + my $tmp = eval { $idx->ipc_do('get_terms', 'K', $num) }; + if ($@) { warn "#$num get_terms: $@" } + else { @$kw{keys %$tmp} = values(%$tmp) }; + } + ($docids, [ sort keys %$kw ]); +} + sub add_eml { my ($self, $eml, $vmd, $xoids) = @_; my $im = $self->{-fake_im} // $self->importer; # may create new epoch @@ -385,7 +351,11 @@ sub add_eml { if ($vmd && $vmd->{sync_info}) { set_sync_info($self, $smsg->{blob}, @{$vmd->{sync_info}}); } - $im_mark or return; # duplicate blob returns undef + unless ($im_mark) { # duplicate blob returns undef + return unless wantarray; + my @docids = $oidx->blob_exists($smsg->{blob}); + return _docids_and_maybe_kw $self, \@docids; + } local $self->{current_info} = $smsg->{blob}; my $vivify_xvmd = delete($smsg->{-vivify_xvmd}) // []; # exact matches @@ -419,7 +389,7 @@ sub add_eml { } _add_vmd($self, $idx, $docid, $vmd) if $vmd; } - $vivify_xvmd; + _docids_and_maybe_kw $self, $vivify_xvmd; } elsif (my @docids = _docids_for($self, $eml)) { # fuzzy match from within lei/store for my $docid (@docids) { @@ -429,8 +399,8 @@ sub add_eml { $idx->ipc_do('add_eidx_info', $docid, '.', $eml); _add_vmd($self, $idx, $docid, $vmd) if $vmd; } - \@docids; - } else { # totally new message + _docids_and_maybe_kw $self, \@docids; + } else { # totally new message, no keywords delete $smsg->{-oidx}; # for IPC-friendliness $smsg->{num} = $oidx->adj_counter('eidx_docid', '+'); $oidx->add_overview($eml, $smsg); @@ -438,7 +408,7 @@ sub add_eml { my $idx = $eidx->idx_shard($smsg->{num}); $idx->index_eml($eml, $smsg); _add_vmd($self, $idx, $smsg->{num}, $vmd) if $vmd; - $smsg; + wantarray ? ($smsg, []) : $smsg; } } @@ -557,7 +527,7 @@ sub xchg_stderr { return unless -e $dir; my $old = delete $self->{-tmp_err}; my $pfx = POSIX::strftime('%Y%m%d%H%M%S', gmtime(time)); - my $err = File::Temp->new(TEMPLATE => "$pfx.$$.lei_storeXXXX", + my $err = File::Temp->new(TEMPLATE => "$pfx.$$.err-XXXX", SUFFIX => '.err', DIR => $dir); open STDERR, '>>', $err->filename or die "dup2: $!"; STDERR->autoflush(1); # shared with shard subprocesses @@ -579,10 +549,6 @@ sub done { $self->{priv_eidx}->done; # V2Writable::done xchg_stderr($self); die $err if $err; - - # notify clients ->done has been issued - defined($sock_ref) and - $self->{s2d_op_p}->pkt_do('sto_done_complete', $sock_ref); } sub ipc_atfork_child { @@ -597,48 +563,39 @@ sub ipc_atfork_child { $self->SUPER::ipc_atfork_child; } +sub recv_and_run { + my ($self, @args) = @_; + local $PublicInbox::DS::in_loop = 0; # waitpid synchronously + $self->SUPER::recv_and_run(@args); +} + +sub _sto_atexit { # dwaitpid callback + my ($args, $pid) = @_; + my $self = $args->[0]; + warn "lei/store PID:$pid died \$?=$?\n" if $?; +} + sub write_prepare { my ($self, $lei) = @_; $lei // die 'BUG: $lei not passed'; unless ($self->{-ipc_req}) { - # s2d => store-to-daemon messages - require PublicInbox::PktOp; - my ($s2d_op_c, $s2d_op_p) = PublicInbox::PktOp->pair; my $dir = $lei->store_path; - $self->ipc_lock_init("$dir/ipc.lock"); substr($dir, -length('/lei/store'), 10, ''); pipe(my ($r, $w)) or die "pipe: $!"; $w->autoflush(1); # Mail we import into lei are private, so headers filtered out # by -mda for public mail are not appropriate local @PublicInbox::MDA::BAD_HEADERS = (); - $self->ipc_worker_spawn("lei/store $dir", $lei->oldset, { + $self->wq_workers_start("lei/store $dir", 1, $lei->oldset, { lei => $lei, -err_wr => $w, - to_close => [ $r, $s2d_op_c->{sock} ], - s2d_op_p => $s2d_op_p, + to_close => [ $r ], }); + $self->wq_wait_async(\&_sto_atexit); # outlives $lei require PublicInbox::LeiStoreErr; PublicInbox::LeiStoreErr->new($r, $lei); - $s2d_op_c->{ops} = { - sto_done_complete => [ $lei->can('sto_done_complete') ] - }; } $lei->{sto} = $self; } -# TODO: support SHA-256 -sub git_blob_id { # called via LEI->git_blob_id - my ($self, $eml) = @_; - $eml->header_set($_) for @PublicInbox::Import::UNWANTED_HEADERS; - git_sha(1, $eml)->hexdigest; -} - -# called by lei-daemon before lei->refresh_watches -sub add_sync_folders { - my ($self, @folders) = @_; - my $lms = _lms_rw($self); - for my $f (@folders) { $lms->fid_for($f, 1) } -} - 1;