use strict;
use v5.10.1;
use parent qw(PublicInbox::Lock PublicInbox::IPC);
-use PublicInbox::SearchIdx qw(crlf_adjust);
use PublicInbox::ExtSearchIdx;
use PublicInbox::Import;
-use PublicInbox::InboxWritable;
+use PublicInbox::InboxWritable qw(eml_from_path);
use PublicInbox::V2Writable;
-use PublicInbox::ContentHash qw(content_hash content_digest);
-use PublicInbox::MID qw(mids mids_in);
+use PublicInbox::ContentHash qw(content_hash);
+use PublicInbox::MID qw(mids);
use PublicInbox::LeiSearch;
+use PublicInbox::MDA;
use List::Util qw(max);
sub new {
my (undef, $dir, $opt) = @_;
my $eidx = PublicInbox::ExtSearchIdx->new($dir, $opt);
my $self = bless { priv_eidx => $eidx }, __PACKAGE__;
- eidx_init($self) if $opt->{creat};
+ eidx_init($self)->done if $opt->{creat};
$self;
}
$eidx;
}
-# when a message has no Message-IDs at all, this is needed for
-# unsent Draft messages, at least
-sub _fake_mid_for ($$) {
- my ($eml, $dig) = @_;
- my $mids = mids_in($eml, qw(X-Alt-Message-ID Resent-Message-ID));
- $eml->{-lei_fake_mid} =
- $mids->[0] // PublicInbox::Import::digest2mid($dig, $eml);
-}
-
sub _docids_for ($$) {
my ($self, $eml) = @_;
my %docids;
- my $dig = content_digest($eml);
- my $chash = $dig->clone->digest;
+ my ($chash, $mids) = PublicInbox::LeiSearch::content_key($eml);
my $eidx = eidx_init($self);
my $oidx = $eidx->{oidx};
my $im = $self->{im};
- my $mids = mids($eml);
- $mids->[0] //= _fake_mid_for($eml, $dig);
for my $mid (@$mids) {
my ($id, $prev);
while (my $cur = $oidx->next_by_mid($mid, \$id, \$prev)) {
sort { $a <=> $b } values %docids;
}
-sub set_eml_keywords {
- my ($self, $eml, @kw) = @_;
+sub set_eml_vmd {
+ my ($self, $eml, $vmd) = @_;
my $eidx = eidx_init($self);
my @docids = _docids_for($self, $eml);
for my $docid (@docids) {
- $eidx->idx_shard($docid)->shard_set_keywords($docid, @kw);
+ $eidx->idx_shard($docid)->ipc_do('set_vmd', $docid, $vmd);
}
\@docids;
}
-sub add_eml_keywords {
- my ($self, $eml, @kw) = @_;
+sub add_eml_vmd {
+ my ($self, $eml, $vmd) = @_;
my $eidx = eidx_init($self);
my @docids = _docids_for($self, $eml);
for my $docid (@docids) {
- $eidx->idx_shard($docid)->shard_add_keywords($docid, @kw);
+ $eidx->idx_shard($docid)->ipc_do('add_vmd', $docid, $vmd);
}
\@docids;
}
-sub remove_eml_keywords {
- my ($self, $eml, @kw) = @_;
+sub remove_eml_vmd {
+ my ($self, $eml, $vmd) = @_;
my $eidx = eidx_init($self);
my @docids = _docids_for($self, $eml);
for my $docid (@docids) {
- $eidx->idx_shard($docid)->shard_remove_keywords($docid, @kw);
+ $eidx->idx_shard($docid)->ipc_do('remove_vmd', $docid, $vmd);
}
\@docids;
}
-# cf: https://doc.dovecot.org/configuration_manual/mail_location/mbox/
-my %status2kw = (F => 'flagged', A => 'answered', R => 'seen', T => 'draft');
-# O (old/non-recent), and D (deleted) aren't in JMAP,
-# so probably won't be supported by us.
-sub mbox_keywords {
- my $eml = $_[-1];
- my $s = "@{[$eml->header_raw('X-Status'),$eml->header_raw('Status')]}";
- my %kw;
- $s =~ s/([FART])/$kw{$status2kw{$1}} = 1/sge;
- sort(keys %kw);
-}
-
-# cf: https://cr.yp.to/proto/maildir.html
-my %c2kw = ('D' => 'draft', F => 'flagged', R => 'answered', S => 'seen');
-sub maildir_keywords {
- $_[-1] =~ /:2,([A-Z]+)\z/i ?
- sort(map { $c2kw{$_} // () } split(//, $1)) : ();
-}
-
sub add_eml {
- my ($self, $eml, @kw) = @_;
- my $eidx = eidx_init($self);
+ my ($self, $eml, $vmd) = @_;
+ my $im = $self->importer; # may create new epoch
+ my $eidx = eidx_init($self); # writes ALL.git/objects/info/alternates
my $oidx = $eidx->{oidx};
my $smsg = bless { -oidx => $oidx }, 'PublicInbox::Smsg';
- my $im = $self->importer;
$im->add($eml, undef, $smsg) or return; # duplicate returns undef
- my $msgref = delete $smsg->{-raw_email};
- $smsg->{bytes} = $smsg->{raw_bytes} + crlf_adjust($$msgref);
local $self->{current_info} = $smsg->{blob};
if (my @docids = _docids_for($self, $eml)) {
for my $docid (@docids) {
my $idx = $eidx->idx_shard($docid);
$oidx->add_xref3($docid, -1, $smsg->{blob}, '.');
- $idx->shard_add_eidx_info($docid, '.', $eml); # List-Id
- $idx->shard_add_keywords($docid, @kw) if @kw;
+ # add_eidx_info for List-Id
+ $idx->ipc_do('add_eidx_info', $docid, '.', $eml);
+ $idx->ipc_do('add_vmd', $docid, $vmd) if $vmd;
}
\@docids;
} else {
$oidx->add_overview($eml, $smsg);
$oidx->add_xref3($smsg->{num}, -1, $smsg->{blob}, '.');
my $idx = $eidx->idx_shard($smsg->{num});
- $idx->index_raw($msgref, $eml, $smsg);
- $idx->shard_add_keywords($smsg->{num}, @kw) if @kw;
+ $idx->index_eml($eml, $smsg);
+ $idx->ipc_do('add_vmd', $smsg->{num}, $vmd ) if $vmd;
$smsg;
}
}
sub set_eml {
- my ($self, $eml, @kw) = @_;
- add_eml($self, $eml, @kw) // set_eml_keywords($self, $eml, @kw);
+ my ($self, $eml, $vmd) = @_;
+ add_eml($self, $eml, $vmd) // set_eml_vmd($self, $eml, $vmd);
+}
+
+sub add_eml_maybe {
+ my ($self, $eml) = @_;
+ my $lxs = $self->{lxs_all_local} // die 'BUG: no {lxs_all_local}';
+ return if $lxs->xids_for($eml, 1);
+ add_eml($self, $eml);
+}
+
+# set or update keywords for external message, called via ipc_do
+sub set_xkw {
+ my ($self, $eml, $kw) = @_;
+ my $lxs = $self->{lxs_all_local} // die 'BUG: no {lxs_all_local}';
+ if ($lxs->xids_for($eml, 1)) { # is it in a local external?
+ # TODO: index keywords only
+ } else {
+ set_eml($self, $eml, { kw => $kw });
+ }
+}
+
+sub checkpoint {
+ my ($self, $wait) = @_;
+ if (my $im = $self->{im}) {
+ $wait ? $im->barrier : $im->checkpoint;
+ }
+ $self->{priv_eidx}->checkpoint($wait);
}
sub done {
die $err if $err;
}
+sub ipc_atfork_child {
+ my ($self) = @_;
+ my $lei = $self->{lei};
+ $lei->lei_atfork_child(1) if $lei;
+ $self->SUPER::ipc_atfork_child;
+}
+
+sub refresh_local_externals {
+ my ($self) = @_;
+ my $cfg = $self->{lei}->_lei_cfg or return;
+ my $cur_cfg = $self->{cur_cfg} // -1;
+ my $lxs = $self->{lxs_all_local};
+ if ($cfg != $cur_cfg || !$lxs) {
+ $lxs = PublicInbox::LeiXSearch->new;
+ my @loc = $self->{lei}->externals_each;
+ for my $loc (@loc) { # locals only
+ $lxs->prepare_external($loc) if -d $loc;
+ }
+ $self->{lei}->ale->refresh_externals($lxs);
+ $lxs->{git} = $self->{lei}->ale->git;
+ $self->{lxs_all_local} = $lxs;
+ $self->{cur_cfg} = $cfg;
+ }
+}
+
+sub write_prepare {
+ my ($self, $lei) = @_;
+ unless ($self->{-ipc_req}) {
+ require PublicInbox::LeiXSearch;
+ $self->ipc_lock_init($lei->store_path . '/ipc.lock');
+ # Mail we import into lei are private, so headers filtered out
+ # by -mda for public mail are not appropriate
+ local @PublicInbox::MDA::BAD_HEADERS = ();
+ $self->ipc_worker_spawn('lei_store', $lei->oldset,
+ { lei => $lei });
+ }
+ my $wait = $self->ipc_do('refresh_local_externals');
+ $lei->{sto} = $self;
+}
+
1;