lei_store: handle messages without Message-ID at all

For personal mail, unsent drafts messages are a common source of
messages without Message-IDs.

diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 43fddf6deb8176e16cdc29187118b94e05d45dfc..c8b9d75e909cab717ec58ebc05fb4906c53d7da5 100644 (file)
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -15,8 +15,8 @@ use PublicInbox::ExtSearchIdx;
 use PublicInbox::Import;
 use PublicInbox::InboxWritable;
 use PublicInbox::V2Writable;
-use PublicInbox::ContentHash qw(content_hash);
-use PublicInbox::MID qw(mids);
+use PublicInbox::ContentHash qw(content_hash content_digest);
+use PublicInbox::MID qw(mids mids_in);
 use PublicInbox::LeiSearch;
 use List::Util qw(max);
 
@@ -107,14 +107,26 @@ sub eidx_init {
        $eidx;
 }
 
+# when a message has no Message-IDs at all, this is needed for
+# unsent Draft messages, at least
+sub _fake_mid_for ($$) {
+       my ($eml, $dig) = @_;
+       my $mids = mids_in($eml, qw(X-Alt-Message-ID Resent-Message-ID));
+       $eml->{-lei_fake_mid} =
+               $mids->[0] // PublicInbox::Import::digest2mid($dig, $eml);
+}
+
 sub _docids_for ($$) {
        my ($self, $eml) = @_;
        my %docids;
-       my $chash = content_hash($eml);
+       my $dig = content_digest($eml);
+       my $chash = $dig->clone->digest;
        my $eidx = eidx_init($self);
        my $oidx = $eidx->{oidx};
        my $im = $self->{im};
-       for my $mid (@{mids($eml)}) {
+       my $mids = mids($eml);
+       $mids->[0] //= _fake_mid_for($eml, $dig);
+       for my $mid (@$mids) {
                my ($id, $prev);
                while (my $cur = $oidx->next_by_mid($mid, \$id, \$prev)) {
                        my $oid = $cur->{blob};

diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index bc2e3ef40b0b3980fe8b7de4d8bdd572f276232c..dad3966d9c8ef3b9f7901998cdf0a40369878d40 100644 (file)
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -284,6 +284,8 @@ sub add_overview {
        $smsg->{lines} = $eml->body_raw =~ tr!\n!\n!;
        my $mids = mids_for_index($eml);
        my $refs = parse_references($smsg, $eml, $mids);
+       $mids->[0] //= $smsg->{mid} //= $eml->{-lei_fake_mid};
+       $smsg->{mid} //= '';
        my $subj = $smsg->{subject};
        my $xpath;
        if ($subj ne '') {

diff --git a/lib/PublicInbox/Smsg.pm b/lib/PublicInbox/Smsg.pm
index 140865389bce65102b9a00ee2c739780d4caefb5..9db10c6453ebfefecc12204aa7c1571757eb6cc4 100644 (file)
--- a/lib/PublicInbox/Smsg.pm
+++ b/lib/PublicInbox/Smsg.pm
@@ -69,7 +69,7 @@ sub psgi_cull ($) {
        $self;
 }
 
-# for Import and v1 non-SQLite WWW code paths
+# used for v2, Import and v1 non-SQLite WWW code paths
 sub populate {
        my ($self, $hdr, $sync) = @_;
        for my $f (qw(From To Cc Subject)) {
@@ -100,9 +100,7 @@ sub populate {
        $self->{-ts} = [ my @ts = msg_timestamp($hdr, $sync->{cotime}) ];
        $self->{ds} //= $ds[0]; # no zone
        $self->{ts} //= $ts[0];
-
-       # for v1 users w/o SQLite
-       $self->{mid} //= eval { mids($hdr)->[0] } // '';
+       $self->{mid} //= mids($hdr)->[0];
 }
 
 # no strftime, that is locale-dependent and not for RFC822

diff --git a/t/lei_store.t b/t/lei_store.t
index bc0d66c20823401a8b8c8c61a4f6be1fd1d9c54b..beb5a8c4da0f1e4829fc5d6466541a5a71d07cbd 100644 (file)
--- a/t/lei_store.t
+++ b/t/lei_store.t
@@ -100,6 +100,30 @@ for my $parallel (0, 1) {
 SKIP: {
        require_mods(qw(Storable), 1);
        ok($lst->can('ipc_do'), 'ipc_do works if we have Storable');
+       $eml->header_set('Message-ID', '<ipc-test@example>');
+       my $pid = $lst->ipc_worker_spawn('lei-store');
+       ok($pid > 0, 'got a worker');
+       my $smsg = $lst->ipc_do('set_eml', $eml, qw(seen));
+       is(ref($smsg), 'PublicInbox::Smsg', 'set_eml works over ipc');
+       my $ids = $lst->ipc_do('set_eml', $eml, qw(seen));
+       is_deeply($ids, [ $smsg->{num} ], 'docid returned');
+
+       $eml->header_set('Message-ID');
+       my $no_mid = $lst->ipc_do('set_eml', $eml, qw(seen));
+       my $wait = $lst->ipc_do('done');
+       my @kw = $lst->search->msg_keywords($no_mid->{num});
+       is_deeply(\@kw, [qw(seen)], 'ipc set changed kw');
+
+       is(ref($smsg), 'PublicInbox::Smsg', 'no mid works ipc');
+       $ids = $lst->ipc_do('set_eml', $eml, qw(seen));
+       is_deeply($ids, [ $no_mid->{num} ], 'docid returned w/o mid w/ ipc');
+       $lst->ipc_do('done');
+       $lst->ipc_worker_stop;
+       $ids = $lst->ipc_do('set_eml', $eml, qw(seen answered));
+       is_deeply($ids, [ $no_mid->{num} ], 'docid returned w/o mid w/o ipc');
+       $wait = $lst->ipc_do('done');
+       @kw = $lst->search->msg_keywords($no_mid->{num});
+       is_deeply(\@kw, [qw(answered seen)], 'set changed kw w/o ipc');
 }
 
 done_testing;