X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FLeiStore.pm;h=094e1555a3fcafe8b92d03c8c1b2f15b391d105e;hb=842e684f0a4154787274843eb3c9be2eef11b160;hp=a17c7bab423d26c90055a54fe6b0fa0731707bdc;hpb=46632639a56dfca600d103f548b561316fc8b326;p=public-inbox.git diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm index a17c7bab..094e1555 100644 --- a/lib/PublicInbox/LeiStore.pm +++ b/lib/PublicInbox/LeiStore.pm @@ -1,33 +1,39 @@ -# Copyright (C) 2020 all contributors +# Copyright (C) 2020-2021 all contributors # License: AGPL-3.0+ # # Local storage (cache/memo) for lei(1), suitable for personal/private # mail iff on encrypted device/FS. Based on v2, but only deduplicates -# based on git OID. +# git storage based on git OID (index deduplication is done in ContentHash) # # for xref3, the following are constant: $eidx_key = '.', $xnum = -1 +# +# We rely on the synchronous IPC API for this in lei-daemon and +# multiple lei clients to write to it at once. This allows the +# lei/store IPC process to be decoupled from network latency in +# lei WQ workers. package PublicInbox::LeiStore; use strict; use v5.10.1; -use parent qw(PublicInbox::Lock); -use PublicInbox::SearchIdx qw(crlf_adjust); +use parent qw(PublicInbox::Lock PublicInbox::IPC); use PublicInbox::ExtSearchIdx; +use PublicInbox::Eml; use PublicInbox::Import; -use PublicInbox::InboxWritable; +use PublicInbox::InboxWritable qw(eml_from_path); use PublicInbox::V2Writable; use PublicInbox::ContentHash qw(content_hash); use PublicInbox::MID qw(mids); use PublicInbox::LeiSearch; +use PublicInbox::MDA; +use PublicInbox::Spawn qw(spawn); use List::Util qw(max); +use File::Temp (); +use POSIX (); sub new { my (undef, $dir, $opt) = @_; my $eidx = PublicInbox::ExtSearchIdx->new($dir, $opt); my $self = bless { priv_eidx => $eidx }, __PACKAGE__; - if ($opt->{creat}) { - PublicInbox::SearchIdx::load_xapian_writable(); - eidx_init($self); - } + eidx_init($self)->done if $opt->{creat}; $self; } @@ -54,30 +60,13 @@ sub git_epoch_max { sub git_ident ($) { my ($git) = @_; - chomp(my $i = $git->qx(qw(var GIT_COMMITTER_IDENT))); - warn "$git->{git_dir} GIT_COMMITTER_IDENT failed\n" if $?; + my $rdr = {}; + open $rdr->{2}, '>', '/dev/null' or die "open /dev/null: $!"; + chomp(my $i = $git->qx([qw(var GIT_COMMITTER_IDENT)], undef, $rdr)); $i =~ /\A(.+) <([^>]+)> [0-9]+ [-\+]?[0-9]+$/ ? ($1, $2) : ('lei user', 'x@example.com') } -# We will support users combining storage across multiple machines -# somehow. Use per-machine refnames to make it easy-to-identify -# where a message came from -sub host_head () { - state $h = do { - my $x = PublicInbox::ExtSearchIdx::host_ident; - # Similar rules found in git.git/remote.c::valid_remote_nick - # and git.git/refs.c::check_refname_component - $x =~ s!(?:\.lock|/)+\z!!gs; # must not end with ".lock" or "/" - $x =~ tr/././s; # no dot-dot, collapse them - $x =~ s/@\{/\@-/gs; - $x =~ s/\A\./-/s; - # no "*", ":", "?", "[", "\", "^", "~", SP, TAB; "]" is OK - $x =~ tr^a-zA-Z0-9!"#$%&'()+,\-.;<=>@]_`{|}^-^c; - $x - }; -} - sub importer { my ($self) = @_; my $max; @@ -96,9 +85,12 @@ sub importer { while (1) { my $latest = "$pfx/$max.git"; my $old = -e $latest; - PublicInbox::Import::init_bare($latest, host_head); + PublicInbox::Import::init_bare($latest); my $git = PublicInbox::Git->new($latest); - $git->qx(qw(config core.sharedRepository 0600)) if !$old; + if (!$old) { + $git->qx(qw(config core.sharedRepository 0600)); + $self->done; # force eidx_init on next round + } my $packed_bytes = $git->packed_bytes; my $unpacked_bytes = $packed_bytes / $self->packing_factor; if ($unpacked_bytes >= $self->rotate_bytes) { @@ -110,7 +102,6 @@ sub importer { $im->{bytes_added} = int($packed_bytes / $self->packing_factor); $im->{lock_path} = undef; $im->{path_type} = 'v2'; - $im->{'ref'} = host_head; return $im; } } @@ -119,23 +110,33 @@ sub search { PublicInbox::LeiSearch->new($_[0]->{priv_eidx}->{topdir}); } +# follows the stderr file +sub _tail_err { + my ($self) = @_; + print { $self->{-err_wr} } readline($self->{-tmp_err}); +} + sub eidx_init { my ($self) = @_; my $eidx = $self->{priv_eidx}; + my $tl = wantarray && $self->{-err_wr} ? + PublicInbox::OnDestroy->new($$, \&_tail_err, $self) : + undef; $eidx->idx_init({-private => 1}); - $eidx; + wantarray ? ($eidx, $tl) : $eidx; } sub _docids_for ($$) { my ($self, $eml) = @_; my %docids; - my $chash = content_hash($eml); - my $eidx = eidx_init($self); + my $eidx = $self->{priv_eidx}; + my ($chash, $mids) = PublicInbox::LeiSearch::content_key($eml); my $oidx = $eidx->{oidx}; my $im = $self->{im}; - for my $mid (@{mids($eml)}) { + for my $mid (@$mids) { my ($id, $prev); while (my $cur = $oidx->next_by_mid($mid, \$id, \$prev)) { + next if $cur->{bytes} == 0; # external-only message my $oid = $cur->{blob}; my $docid = $cur->{num}; my $bref = $im ? $im->cat_blob($oid) : undef; @@ -151,82 +152,213 @@ sub _docids_for ($$) { sort { $a <=> $b } values %docids; } -sub set_eml_keywords { - my ($self, $eml, @kw) = @_; - my $eidx = eidx_init($self); - my @docids = _docids_for($self, $eml); - for my $docid (@docids) { - $eidx->idx_shard($docid)->shard_set_keywords($docid, @kw); +sub set_eml_vmd { + my ($self, $eml, $vmd, $docids) = @_; + my ($eidx, $tl) = eidx_init($self); + $docids //= [ _docids_for($self, $eml) ]; + for my $docid (@$docids) { + $eidx->idx_shard($docid)->ipc_do('set_vmd', $docid, $vmd); } - \@docids; + $docids; } -sub add_eml_keywords { - my ($self, $eml, @kw) = @_; - my $eidx = eidx_init($self); +sub add_eml_vmd { + my ($self, $eml, $vmd) = @_; + my ($eidx, $tl) = eidx_init($self); my @docids = _docids_for($self, $eml); for my $docid (@docids) { - $eidx->idx_shard($docid)->shard_add_keywords($docid, @kw); + $eidx->idx_shard($docid)->ipc_do('add_vmd', $docid, $vmd); } \@docids; } -sub remove_eml_keywords { - my ($self, $eml, @kw) = @_; - my $eidx = eidx_init($self); +sub remove_eml_vmd { + my ($self, $eml, $vmd) = @_; + my ($eidx, $tl) = eidx_init($self); my @docids = _docids_for($self, $eml); for my $docid (@docids) { - $eidx->idx_shard($docid)->shard_remove_keywords($docid, @kw); + $eidx->idx_shard($docid)->ipc_do('remove_vmd', $docid, $vmd); } \@docids; } -# cf: https://doc.dovecot.org/configuration_manual/mail_location/mbox/ -my %status2kw = (F => 'flagged', A => 'answered', R => 'seen', T => 'draft'); -# O (old/non-recent), and D (deleted) aren't in JMAP, -# so probably won't be supported by us. -sub mbox_keywords { - my $eml = $_[-1]; - my $s = "@{[$eml->header_raw('X-Status'),$eml->header_raw('Status')]}"; - my %kw; - $s =~ s/([FART])/$kw{$status2kw{$1}} = 1/sge; - sort(keys %kw); -} - -# cf: https://cr.yp.to/proto/maildir.html -my %c2kw = ('D' => 'draft', F => 'flagged', R => 'answered', S => 'seen'); -sub maildir_keywords { - $_[-1] =~ /:2,([A-Z]+)\z/i ? - sort(map { $c2kw{$_} // () } split(//, $1)) : (); -} - sub add_eml { - my ($self, $eml, @kw) = @_; - my $eidx = eidx_init($self); - my $oidx = $eidx->{oidx}; + my ($self, $eml, $vmd, $xoids) = @_; + my $im = $self->importer; # may create new epoch + my ($eidx, $tl) = eidx_init($self); # updates/writes alternates file + my $oidx = $eidx->{oidx}; # PublicInbox::Import::add checks this my $smsg = bless { -oidx => $oidx }, 'PublicInbox::Smsg'; - my $im = $self->importer; $im->add($eml, undef, $smsg) or return; # duplicate returns undef - my $msgref = delete $smsg->{-raw_email}; - $smsg->{bytes} = $smsg->{raw_bytes} + crlf_adjust($$msgref); local $self->{current_info} = $smsg->{blob}; - if (my @docids = _docids_for($self, $eml)) { + my $vivify_xvmd = delete($smsg->{-vivify_xvmd}) // []; # exact matches + if ($xoids) { # fuzzy matches from externals in ale->xoids_for + delete $xoids->{$smsg->{blob}}; # added later + if (scalar keys %$xoids) { + my %docids = map { $_ => 1 } @$vivify_xvmd; + for my $oid (keys %$xoids) { + my @id = $oidx->blob_exists($oid); + @docids{@id} = @id; + } + @$vivify_xvmd = sort { $a <=> $b } keys(%docids); + } + } + if (@$vivify_xvmd) { + $xoids //= {}; + $xoids->{$smsg->{blob}} = 1; + for my $docid (@$vivify_xvmd) { + my $cur = $oidx->get_art($docid); + my $idx = $eidx->idx_shard($docid); + if (!$cur || $cur->{bytes} == 0) { # really vivifying + $smsg->{num} = $docid; + $oidx->add_overview($eml, $smsg); + $smsg->{-merge_vmd} = 1; + $idx->index_eml($eml, $smsg); + } else { # lse fuzzy hit off ale + $idx->ipc_do('add_eidx_info', $docid, '.', $eml); + } + for my $oid (keys %$xoids) { + $oidx->add_xref3($docid, -1, $oid, '.'); + } + $idx->ipc_do('add_vmd', $docid, $vmd) if $vmd; + } + $vivify_xvmd; + } elsif (my @docids = _docids_for($self, $eml)) { + # fuzzy match from within lei/store for my $docid (@docids) { my $idx = $eidx->idx_shard($docid); $oidx->add_xref3($docid, -1, $smsg->{blob}, '.'); - $idx->shard_add_eidx_info($docid, '.', $eml); # List-Id - $idx->shard_add_keywords($docid, @kw) if @kw; + # add_eidx_info for List-Id + $idx->ipc_do('add_eidx_info', $docid, '.', $eml); + $idx->ipc_do('add_vmd', $docid, $vmd) if $vmd; } - } else { + \@docids; + } else { # totally new message $smsg->{num} = $oidx->adj_counter('eidx_docid', '+'); $oidx->add_overview($eml, $smsg); $oidx->add_xref3($smsg->{num}, -1, $smsg->{blob}, '.'); my $idx = $eidx->idx_shard($smsg->{num}); - $idx->index_raw($msgref, $eml, $smsg); - $idx->shard_add_keywords($smsg->{num}, @kw) if @kw; + $idx->index_eml($eml, $smsg); + $idx->ipc_do('add_vmd', $smsg->{num}, $vmd) if $vmd; + $smsg; + } +} + +sub set_eml { + my ($self, $eml, $vmd, $xoids) = @_; + add_eml($self, $eml, $vmd, $xoids) // + set_eml_vmd($self, $eml, $vmd); +} + +sub _external_only ($$$) { + my ($self, $xoids, $eml) = @_; + my $eidx = $self->{priv_eidx}; + my $oidx = $eidx->{oidx} // die 'BUG: {oidx} missing'; + my $smsg = bless { blob => '' }, 'PublicInbox::Smsg'; + $smsg->{num} = $oidx->adj_counter('eidx_docid', '+'); + # save space for an externals-only message + my $hdr = $eml->header_obj; + $smsg->populate($hdr); # sets lines == 0 + $smsg->{bytes} = 0; + delete @$smsg{qw(From Subject)}; + $smsg->{to} = $smsg->{cc} = $smsg->{from} = ''; + $oidx->add_overview($hdr, $smsg); # subject+references for threading + $smsg->{subject} = ''; + for my $oid (keys %$xoids) { + $oidx->add_xref3($smsg->{num}, -1, $oid, '.'); + } + my $idx = $eidx->idx_shard($smsg->{num}); + $idx->index_eml(PublicInbox::Eml->new("\n\n"), $smsg); + ($smsg, $idx); +} + +sub update_xvmd { + my ($self, $xoids, $eml, $vmd_mod) = @_; + my ($eidx, $tl) = eidx_init($self); + my $oidx = $eidx->{oidx}; + my %seen; + for my $oid (keys %$xoids) { + my @docids = $oidx->blob_exists($oid) or next; + scalar(@docids) > 1 and + warn "W: $oid indexed as multiple docids: @docids\n"; + for my $docid (@docids) { + next if $seen{$docid}++; + my $idx = $eidx->idx_shard($docid); + $idx->ipc_do('update_vmd', $docid, $vmd_mod); + } + delete $xoids->{$oid}; + } + return unless scalar(keys(%$xoids)); + + # see if it was indexed, but with different OID(s) + if (my @docids = _docids_for($self, $eml)) { + for my $docid (@docids) { + next if $seen{$docid}; + for my $oid (keys %$xoids) { + $oidx->add_xref3($docid, -1, $oid, '.'); + } + my $idx = $eidx->idx_shard($docid); + $idx->ipc_do('update_vmd', $docid, $vmd_mod); + } + return; + } + # totally unseen + my ($smsg, $idx) = _external_only($self, $xoids, $eml); + $idx->ipc_do('update_vmd', $smsg->{num}, $vmd_mod); +} + +# set or update keywords for external message, called via ipc_do +sub set_xvmd { + my ($self, $xoids, $eml, $vmd) = @_; + + my ($eidx, $tl) = eidx_init($self); + my $oidx = $eidx->{oidx}; + my %seen; + + # see if we can just update existing docs + for my $oid (keys %$xoids) { + my @docids = $oidx->blob_exists($oid) or next; + scalar(@docids) > 1 and + warn "W: $oid indexed as multiple docids: @docids\n"; + for my $docid (@docids) { + next if $seen{$docid}++; + my $idx = $eidx->idx_shard($docid); + $idx->ipc_do('set_vmd', $docid, $vmd); + } + delete $xoids->{$oid}; # all done with this oid } - $smsg->{blob} + return unless scalar(keys(%$xoids)); + + # n.b. we don't do _docids_for here, we expect the caller + # already checked $lse->kw_changed before calling this sub + + return unless (@{$vmd->{kw} // []}) || (@{$vmd->{L} // []}); + # totally unseen: + my ($smsg, $idx) = _external_only($self, $xoids, $eml); + $idx->ipc_do('add_vmd', $smsg->{num}, $vmd); +} + +sub checkpoint { + my ($self, $wait) = @_; + if (my $im = $self->{im}) { + $wait ? $im->barrier : $im->checkpoint; + } + $self->{priv_eidx}->checkpoint($wait); +} + +sub xchg_stderr { + my ($self) = @_; + _tail_err($self) if $self->{-err_wr}; + my $dir = $self->{priv_eidx}->{topdir}; + return unless -e $dir; + my $old = delete $self->{-tmp_err}; + my $pfx = POSIX::strftime('%Y%m%d%H%M%S', gmtime(time)); + my $err = File::Temp->new(TEMPLATE => "$pfx.$$.lei_storeXXXX", + SUFFIX => '.err', DIR => $dir); + open STDERR, '>>', $err->filename or die "dup2: $!"; + STDERR->autoflush(1); # shared with shard subprocesses + $self->{-tmp_err} = $err; # separate file description for RO access + undef; } sub done { @@ -239,8 +371,46 @@ sub done { warn $err; } } - $self->{priv_eidx}->done; + $self->{priv_eidx}->done; # V2Writable::done + xchg_stderr($self); die $err if $err; } +sub ipc_atfork_child { + my ($self) = @_; + my $lei = $self->{lei}; + $lei->_lei_atfork_child(1) if $lei; + xchg_stderr($self); + if (my $err = delete($self->{err_pipe})) { + close $err->[0]; + $self->{-err_wr} = $err->[1]; + } + $SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb(); + $self->SUPER::ipc_atfork_child; +} + +sub write_prepare { + my ($self, $lei) = @_; + unless ($self->{-ipc_req}) { + my $d = $lei->store_path; + $self->ipc_lock_init("$d/ipc.lock"); + substr($d, -length('/lei/store'), 10, ''); + my $err_pipe; + unless ($lei->{oneshot}) { + pipe(my ($r, $w)) or die "pipe: $!"; + $err_pipe = [ $r, $w ]; + } + # Mail we import into lei are private, so headers filtered out + # by -mda for public mail are not appropriate + local @PublicInbox::MDA::BAD_HEADERS = (); + $self->ipc_worker_spawn("lei/store $d", $lei->oldset, + { lei => $lei, err_pipe => $err_pipe }); + if ($err_pipe) { + require PublicInbox::LeiStoreErr; + PublicInbox::LeiStoreErr->new($err_pipe->[0], $lei); + } + } + $lei->{sto} = $self; +} + 1;