extsearchidx: enforce -index before -extindex

We cannot set xref3 data without the `xnum' column to
tie it to the per-inbox over.sqlite3 DB.  So ensure we don't
read brand-new history that only exists in git, but instead
rely on last_commit and last_xap15-$EPOCH metadata in msgmap
to decide how far we can index.

Before this change, it was possible to miss messages in
the extindex if -index did not run (which will be fixable by
upcoming --reindex support in -extindex).

diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index b0a12bcabce0e97e04f6e8fb13de346f91439f90..84449cb466457fea9fd9d25d6c3c0f200bd7ac98 100644 (file)
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -291,7 +291,12 @@ sub _sync_inbox ($$$) {
        } elsif ($v == 1) {
                my $uv = $ibx->uidvalidity;
                my $lc = $self->{oidx}->eidx_meta("lc-v1:$ekey//$uv");
-               my $stk = prepare_stack($sync, $lc ? "$lc..HEAD" : 'HEAD');
+               my $head = $ibx->mm->last_commit;
+               unless (defined $head) {
+                       warn "E: $ibx->{inboxdir} is not indexed\n";
+                       return;
+               }
+               my $stk = prepare_stack($sync, $lc ? "$lc..$head" : $head);
                my $unit = { stack => $stk, git => $ibx->git };
                push @{$sync->{todo}}, $unit;
        } else {

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 07a7fa426e6b7a142419f0c3aa6be0b9b464a6f5..bef3a67a7157bbda72cb0b1933f567d0f6ed2292 100644 (file)
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -1073,10 +1073,22 @@ sub sync_prepare ($$) {
                $pfx //= $sync->{ibx}->{inboxdir};
        }
 
-       # reindex stops at the current heads and we later rerun index_sync
-       # without {reindex}
-       my $reindex_heads = $self->last_commits($sync) if $sync->{reindex};
-
+       my $reindex_heads;
+       if ($self->{ibx_map}) {
+               # ExtSearchIdx won't index messages unless they're in
+               # over.sqlite3 for a given inbox, so don't read beyond
+               # what's in the per-inbox index.
+               $reindex_heads = [];
+               my $v = PublicInbox::Search::SCHEMA_VERSION;
+               my $mm = $sync->{ibx}->mm;
+               for my $i (0..$sync->{epoch_max}) {
+                       $reindex_heads->[$i] = $mm->last_commit_xap($v, $i);
+               }
+       } elsif ($sync->{reindex}) { # V2 inbox
+               # reindex stops at the current heads and we later
+               # rerun index_sync without {reindex}
+               $reindex_heads = $self->last_commits($sync);
+       }
        if ($sync->{max_size} = $sync->{-opt}->{max_size}) {
                $sync->{index_oid} = $self->can('index_oid');
        }

diff --git a/t/extsearch.t b/t/extsearch.t
index 965122278e56a777ac31548fb661b7e7ac984abd..70a60b5ae522b90f4c20b1671993af32e3ea6e67 100644 (file)
--- a/t/extsearch.t
+++ b/t/extsearch.t
@@ -176,6 +176,60 @@ is(scalar(@it), 2, 'two inboxes');
 like($it[0]->get_document->get_data, qr/v2test/, 'docdata matched v2');
 like($it[1]->get_document->get_data, qr/v1test/, 'docdata matched v1');
 
+if ('inject w/o indexing') {
+       use PublicInbox::Import;
+       use PublicInbox::Search;
+       my $schema_version = PublicInbox::Search::SCHEMA_VERSION();
+       my $v1ibx = PublicInbox::Config->new->lookup_name('v1test');
+       my $last_v1_commit = $v1ibx->mm->last_commit;
+       my $v2ibx = PublicInbox::Config->new->lookup_name('v2test');
+       my $last_v2_commit = $v2ibx->mm->last_commit_xap($schema_version, 0);
+       my $git0 = PublicInbox::Git->new("$v2ibx->{inboxdir}/git/0.git");
+       chomp(my $cmt = $git0->qx(qw(rev-parse HEAD^0)));
+       is($last_v2_commit, $cmt, 'v2 index up-to-date');
+
+       my $v2im = PublicInbox::Import->new($git0, undef, undef, $v2ibx);
+       $v2im->{lock_path} = undef;
+       $v2im->{path_type} = 'v2';
+       $v2im->add(eml_load('t/mda-mime.eml'));
+       $v2im->done;
+       chomp(my $tip = $git0->qx(qw(rev-parse HEAD^0)));
+       isnt($tip, $cmt, '0.git v2 updated');
+
+       # inject a message w/o updating index
+       rename("$home/v1test/public-inbox", "$home/v1test/skip-index") or
+               BAIL_OUT $!;
+       open(my $eh, '<', 't/iso-2202-jp.eml') or BAIL_OUT $!;
+       run_script(['-mda', '--no-precheck'], $env, { 0 => $eh}) or
+               BAIL_OUT '-mda';
+       rename("$home/v1test/skip-index", "$home/v1test/public-inbox") or
+               BAIL_OUT $!;
+
+       my ($in, $out, $err);
+       $in = $out = $err = '';
+       my $opt = { 0 => \$in, 1 => \$out, 2 => \$err };
+       ok(run_script([qw(-extindex -v -v --all), "$home/extindex"],
+               undef, undef), 'extindex noop');
+       $es->{xdb}->reopen;
+       my $mset = $es->mset('mid:199707281508.AAA24167@hoyogw.example');
+       is($mset->size, 0, 'did not attempt to index unindexed v1 message');
+       $mset = $es->mset('mid:multipart-html-sucks@11');
+       is($mset->size, 0, 'did not attempt to index unindexed v2 message');
+       ok(run_script([qw(-index --all)]), 'indexed v1 and v2 inboxes');
+
+       isnt($v1ibx->mm->last_commit, $last_v1_commit, '-index v1 worked');
+       isnt($v2ibx->mm->last_commit_xap($schema_version, 0),
+               $last_v2_commit, '-index v2 worked');
+       ok(run_script([qw(-extindex --all), "$home/extindex"]),
+               'extindex updates');
+
+       $es->{xdb}->reopen;
+       $mset = $es->mset('mid:199707281508.AAA24167@hoyogw.example');
+       is($mset->size, 1, 'got v1 message');
+       $mset = $es->mset('mid:multipart-html-sucks@11');
+       is($mset->size, 1, 'got v2 message');
+}
+
 if ('remove v1test and test gc') {
        xsys([qw(git config --unset publicinbox.v1test.inboxdir)],
                { GIT_CONFIG => $cfg_path });