X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FLeiStore.pm;h=b76af4d3f2ce861b02d324b2e4711d2a3cc543b0;hb=43c43f785aa53607a0dd050989da5d7fd0dcfff4;hp=f8383d5e5cd91cba83f2b9ddc615927131f11698;hpb=7d36c23938cddd2ce5baabe3ee5b7a579378aaba;p=public-inbox.git diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm index f8383d5e..b76af4d3 100644 --- a/lib/PublicInbox/LeiStore.pm +++ b/lib/PublicInbox/LeiStore.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2020 all contributors +# Copyright (C) 2020-2021 all contributors # License: AGPL-3.0+ # # Local storage (cache/memo) for lei(1), suitable for personal/private @@ -9,25 +9,22 @@ package PublicInbox::LeiStore; use strict; use v5.10.1; -use parent qw(PublicInbox::Lock); -use PublicInbox::SearchIdx qw(crlf_adjust); +use parent qw(PublicInbox::Lock PublicInbox::IPC); use PublicInbox::ExtSearchIdx; use PublicInbox::Import; -use PublicInbox::InboxWritable; +use PublicInbox::InboxWritable qw(eml_from_path); use PublicInbox::V2Writable; use PublicInbox::ContentHash qw(content_hash); use PublicInbox::MID qw(mids); use PublicInbox::LeiSearch; +use PublicInbox::MDA; use List::Util qw(max); sub new { my (undef, $dir, $opt) = @_; my $eidx = PublicInbox::ExtSearchIdx->new($dir, $opt); my $self = bless { priv_eidx => $eidx }, __PACKAGE__; - if ($opt->{creat}) { - PublicInbox::SearchIdx::load_xapian_writable(); - eidx_init($self); - } + eidx_init($self)->done if $opt->{creat}; $self; } @@ -110,13 +107,14 @@ sub eidx_init { sub _docids_for ($$) { my ($self, $eml) = @_; my %docids; - my $chash = content_hash($eml); + my ($chash, $mids) = PublicInbox::LeiSearch::content_key($eml); my $eidx = eidx_init($self); my $oidx = $eidx->{oidx}; my $im = $self->{im}; - for my $mid (@{mids($eml)}) { + for my $mid (@$mids) { my ($id, $prev); while (my $cur = $oidx->next_by_mid($mid, \$id, \$prev)) { + next if $cur->{bytes} == 0; # external-only message my $oid = $cur->{blob}; my $docid = $cur->{num}; my $bref = $im ? $im->cat_blob($oid) : undef; @@ -132,82 +130,206 @@ sub _docids_for ($$) { sort { $a <=> $b } values %docids; } -sub set_eml_keywords { - my ($self, $eml, @kw) = @_; +sub set_eml_vmd { + my ($self, $eml, $vmd) = @_; my $eidx = eidx_init($self); my @docids = _docids_for($self, $eml); for my $docid (@docids) { - $eidx->idx_shard($docid)->shard_set_keywords($docid, @kw); + $eidx->idx_shard($docid)->ipc_do('set_vmd', $docid, $vmd); } \@docids; } -sub add_eml_keywords { - my ($self, $eml, @kw) = @_; +sub add_eml_vmd { + my ($self, $eml, $vmd) = @_; my $eidx = eidx_init($self); my @docids = _docids_for($self, $eml); for my $docid (@docids) { - $eidx->idx_shard($docid)->shard_add_keywords($docid, @kw); + $eidx->idx_shard($docid)->ipc_do('add_vmd', $docid, $vmd); } \@docids; } -sub remove_eml_keywords { - my ($self, $eml, @kw) = @_; +sub remove_eml_vmd { + my ($self, $eml, $vmd) = @_; my $eidx = eidx_init($self); my @docids = _docids_for($self, $eml); for my $docid (@docids) { - $eidx->idx_shard($docid)->shard_remove_keywords($docid, @kw); + $eidx->idx_shard($docid)->ipc_do('remove_vmd', $docid, $vmd); } \@docids; } -# cf: https://doc.dovecot.org/configuration_manual/mail_location/mbox/ -my %status2kw = (F => 'flagged', A => 'answered', R => 'seen', T => 'draft'); -# O (old/non-recent), and D (deleted) aren't in JMAP, -# so probably won't be supported by us. -sub mbox_keywords { - my $eml = $_[-1]; - my $s = "@{[$eml->header_raw('X-Status'),$eml->header_raw('Status')]}"; - my %kw; - $s =~ s/([FART])/$kw{$status2kw{$1}} = 1/sge; - sort(keys %kw); -} - -# cf: https://cr.yp.to/proto/maildir.html -my %c2kw = ('D' => 'draft', F => 'flagged', R => 'answered', S => 'seen'); -sub maildir_keywords { - $_[-1] =~ /:2,([A-Z]+)\z/i ? - sort(map { $c2kw{$_} // () } split(//, $1)) : (); -} - sub add_eml { - my ($self, $eml, @kw) = @_; - my $eidx = eidx_init($self); - my $oidx = $eidx->{oidx}; + my ($self, $eml, $vmd, $xoids) = @_; + my $im = $self->importer; # may create new epoch + my $eidx = eidx_init($self); # writes ALL.git/objects/info/alternates + my $oidx = $eidx->{oidx}; # PublicInbox::Import::add checks this my $smsg = bless { -oidx => $oidx }, 'PublicInbox::Smsg'; - my $im = $self->importer; $im->add($eml, undef, $smsg) or return; # duplicate returns undef - my $msgref = delete $smsg->{-raw_email}; - $smsg->{bytes} = $smsg->{raw_bytes} + crlf_adjust($$msgref); local $self->{current_info} = $smsg->{blob}; - if (my @docids = _docids_for($self, $eml)) { + my $vivify_xvmd = delete($smsg->{-vivify_xvmd}) // []; # exact matches + if ($xoids) { # fuzzy matches from externals in ale->xoids_for + delete $xoids->{$smsg->{blob}}; # added later + if (scalar keys %$xoids) { + my %docids = map { $_ => 1 } @$vivify_xvmd; + for my $oid (keys %$xoids) { + my @id = $oidx->blob_exists($oid); + @docids{@id} = @id; + } + @$vivify_xvmd = sort { $a <=> $b } keys(%docids); + } + } + if (@$vivify_xvmd) { + $xoids //= {}; + $xoids->{$smsg->{blob}} = 1; + for my $docid (@$vivify_xvmd) { + my $cur = $oidx->get_art($docid); + my $idx = $eidx->idx_shard($docid); + if (!$cur || $cur->{bytes} == 0) { # really vivifying + $smsg->{num} = $docid; + $oidx->add_overview($eml, $smsg); + $smsg->{-merge_vmd} = 1; + $idx->index_eml($eml, $smsg); + } else { # lse fuzzy hit off ale + $idx->ipc_do('add_eidx_info', $docid, '.', $eml); + } + for my $oid (keys %$xoids) { + $oidx->add_xref3($docid, -1, $oid, '.'); + } + $idx->ipc_do('add_vmd', $docid, $vmd) if $vmd; + } + $vivify_xvmd; + } elsif (my @docids = _docids_for($self, $eml)) { + # fuzzy match from within lei/store for my $docid (@docids) { my $idx = $eidx->idx_shard($docid); $oidx->add_xref3($docid, -1, $smsg->{blob}, '.'); - $idx->shard_add_eidx_info($docid, '.', $eml); # List-Id - $idx->shard_add_keywords($docid, @kw) if @kw; + # add_eidx_info for List-Id + $idx->ipc_do('add_eidx_info', $docid, '.', $eml); + $idx->ipc_do('add_vmd', $docid, $vmd) if $vmd; } - } else { + \@docids; + } else { # totally new message $smsg->{num} = $oidx->adj_counter('eidx_docid', '+'); $oidx->add_overview($eml, $smsg); $oidx->add_xref3($smsg->{num}, -1, $smsg->{blob}, '.'); my $idx = $eidx->idx_shard($smsg->{num}); - $idx->index_raw($msgref, $eml, $smsg); - $idx->shard_add_keywords($smsg->{num}, @kw) if @kw; + $idx->index_eml($eml, $smsg); + $idx->ipc_do('add_vmd', $smsg->{num}, $vmd) if $vmd; + $smsg; + } +} + +sub set_eml { + my ($self, $eml, $vmd, $xoids) = @_; + add_eml($self, $eml, $vmd, $xoids) // + set_eml_vmd($self, $eml, $vmd); +} + +sub _external_only ($$$) { + my ($self, $xoids, $eml) = @_; + my $eidx = $self->{priv_eidx}; + my $oidx = $eidx->{oidx} // die 'BUG: {oidx} missing'; + my $smsg = bless { blob => '' }, 'PublicInbox::Smsg'; + $smsg->{num} = $oidx->adj_counter('eidx_docid', '+'); + # save space for an externals-only message + my $hdr = $eml->header_obj; + $smsg->populate($hdr); # sets lines == 0 + $smsg->{bytes} = 0; + delete @$smsg{qw(From Subject)}; + $smsg->{to} = $smsg->{cc} = $smsg->{from} = ''; + $oidx->add_overview($hdr, $smsg); # subject+references for threading + $smsg->{subject} = ''; + for my $oid (keys %$xoids) { + $oidx->add_xref3($smsg->{num}, -1, $oid, '.'); + } + my $idx = $eidx->idx_shard($smsg->{num}); + $idx->index_eml(PublicInbox::Eml->new("\n\n"), $smsg); + ($smsg, $idx); +} + +sub update_xvmd { + my ($self, $xoids, $eml, $vmd_mod) = @_; + my $eidx = eidx_init($self); + my $oidx = $eidx->{oidx}; + my %seen; + for my $oid (keys %$xoids) { + my @docids = $oidx->blob_exists($oid) or next; + scalar(@docids) > 1 and + warn "W: $oid indexed as multiple docids: @docids\n"; + for my $docid (@docids) { + next if $seen{$docid}++; + my $idx = $eidx->idx_shard($docid); + $idx->ipc_do('update_vmd', $docid, $vmd_mod); + } + delete $xoids->{$oid}; + } + return unless scalar(keys(%$xoids)); + + # see if it was indexed, but with different OID(s) + if (my @docids = _docids_for($self, $eml)) { + for my $docid (@docids) { + next if $seen{$docid}; + for my $oid (keys %$xoids) { + $oidx->add_xref3($docid, -1, $oid, '.'); + } + my $idx = $eidx->idx_shard($docid); + $idx->ipc_do('update_vmd', $docid, $vmd_mod); + } + return; } - $smsg->{blob} + # totally unseen + my ($smsg, $idx) = _external_only($self, $xoids, $eml); + $idx->ipc_do('update_vmd', $smsg->{num}, $vmd_mod); +} + +# set or update keywords for external message, called via ipc_do +sub set_xvmd { + my ($self, $xoids, $eml, $vmd) = @_; + + my $eidx = eidx_init($self); + my $oidx = $eidx->{oidx}; + my %seen; + + # see if we can just update existing docs + for my $oid (keys %$xoids) { + my @docids = $oidx->blob_exists($oid) or next; + scalar(@docids) > 1 and + warn "W: $oid indexed as multiple docids: @docids\n"; + for my $docid (@docids) { + next if $seen{$docid}++; + my $idx = $eidx->idx_shard($docid); + $idx->ipc_do('set_vmd', $docid, $vmd); + } + delete $xoids->{$oid}; # all done with this oid + } + return unless scalar(keys(%$xoids)); + + # see if it was indexed, but with different OID(s) + if (my @docids = _docids_for($self, $eml)) { + for my $docid (@docids) { + next if $seen{$docid}; + for my $oid (keys %$xoids) { + $oidx->add_xref3($docid, -1, $oid, '.'); + } + my $idx = $eidx->idx_shard($docid); + $idx->ipc_do('set_vmd', $docid, $vmd); + } + return; + } + # totally unseen + my ($smsg, $idx) = _external_only($self, $xoids, $eml); + $idx->ipc_do('add_vmd', $smsg->{num}, $vmd); +} + +sub checkpoint { + my ($self, $wait) = @_; + if (my $im = $self->{im}) { + $wait ? $im->barrier : $im->checkpoint; + } + $self->{priv_eidx}->checkpoint($wait); } sub done { @@ -224,4 +346,26 @@ sub done { die $err if $err; } +sub ipc_atfork_child { + my ($self) = @_; + my $lei = $self->{lei}; + $lei->_lei_atfork_child(1) if $lei; + $self->SUPER::ipc_atfork_child; +} + +sub write_prepare { + my ($self, $lei) = @_; + unless ($self->{-ipc_req}) { + my $d = $lei->store_path; + $self->ipc_lock_init("$d/ipc.lock"); + substr($d, -length('/lei/store'), 10, ''); + # Mail we import into lei are private, so headers filtered out + # by -mda for public mail are not appropriate + local @PublicInbox::MDA::BAD_HEADERS = (); + $self->ipc_worker_spawn("lei/store $d", $lei->oldset, + { lei => $lei }); + } + $lei->{sto} = $self; +} + 1;