lib/PublicInbox/ExtSearchIdx.pm | 13 ++++++++++---
 lib/PublicInbox/OverIdx.pm      | 27 ++++++++++++++++++++++++---
 t/extsearch.t                   | 20 ++++++++++++++++++++
 t/over.t                        |  2 +-

diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index cf90c562ebe27163248ded0ee0138f60d3132e03..d780776fbad5b66ba59e416d0a235c0c929f1e45 100644
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -128,14 +128,21 @@ 	my $idx = $self->idx_shard($docid);
 	my $oid = $req->{oid};
 	my $xibx = $req->{ibx};
 	my $eml = $req->{eml};
+	my $eidx_key = $xibx->eidx_key;
 	if (my $new_smsg = $req->{new_smsg}) { # 'm' on cross-posted message
 		my $xnum = $req->{xnum};
-		$self->{oidx}->add_xref3($docid, $xnum, $oid, $xibx->eidx_key);
+		$self->{oidx}->add_xref3($docid, $xnum, $oid, $eidx_key);
 		$idx->shard_add_eidx_info($docid, $oid, $xibx, $eml);
 		check_batch_limit($req);
 	} else { # 'd'
-		$self->{oidx}->remove_xref3($docid, $oid, $xibx->eidx_key);
-		$idx->shard_remove_eidx_info($docid, $oid, $xibx, $eml);
+		my $rm_eidx_info;
+		my $nr = $self->{oidx}->remove_xref3($docid, $oid, $eidx_key,
+							\$rm_eidx_info);
+		if ($nr == 0) {
+			$idx->shard_remove($oid, $docid);
+		} elsif ($rm_eidx_info) {
+			$idx->shard_remove_eidx_info($docid, $oid, $xibx, $eml);
+		}
 	}
 }
 
diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index 8bec08da0b168746535bd32c445b87207fe72f5e..07cca4e5bb30d9c2053d9bb8ab62788571f2289b 100644
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -595,13 +595,14 @@ 	$sth->bind_param(4, $oidbin, SQL_BLOB);
 	$sth->execute;
 }
 
+# returns remaining reference count to $docid
 sub remove_xref3 {
-	my ($self, $docid, $oidhex, $eidx_key) = @_;
+	my ($self, $docid, $oidhex, $eidx_key, $rm_eidx_info) = @_;
 	begin_lazy($self);
 	my $oidbin = pack('H*', $oidhex);
-	my $sth;
+	my ($sth, $ibx_id);
 	if (defined $eidx_key) {
-		my $ibx_id = id_for($self, 'inboxes', 'ibx_id',
+		$ibx_id = id_for($self, 'inboxes', 'ibx_id',
 					eidx_key => $eidx_key);
 		$sth = $self->{dbh}->prepare_cached(<<'');
 DELETE FROM xref3 WHERE docid = ? AND ibx_id = ? AND oidbin = ?
@@ -617,6 +618,26 @@ 		$sth->bind_param(1, $docid);
 		$sth->bind_param(2, $oidbin, SQL_BLOB);
 	}
 	$sth->execute;
+	$sth = $self->{dbh}->prepare_cached(<<'', undef, 1);
+SELECT COUNT(*) FROM xref3 WHERE docid = ?
+
+	$sth->execute($docid);
+	my $nr = $sth->fetchrow_array;
+	if ($nr == 0) {
+		delete_by_num($self, $docid);
+	} elsif (defined($ibx_id) && $rm_eidx_info) {
+		# if deduplication rules in ContentHash change, it's
+		# possible a docid can have multiple rows with the
+		# same ibx_id.  This governs whether or not we call
+		# ->shard_remove_eidx_info in ExtSearchIdx.
+		$sth = $self->{dbh}->prepare_cached(<<'', undef, 1);
+SELECT COUNT(*) FROM xref3 WHERE docid = ? AND ibx_id = ?
+
+		$sth->execute($docid, $ibx_id);
+		my $count = $sth->fetchrow_array;
+		$$rm_eidx_info = ($count == 0);
+	}
+	$nr;
 }
 
 # for when an xref3 goes missing, this does NOT update {ts}
diff --git a/t/extsearch.t b/t/extsearch.t
index f9f74e5c9db3ad6cf1cb06a477ff3dcb94a28bb3..f585555807f5cc4975115101a1848f010ce3868e 100644
--- a/t/extsearch.t
+++ b/t/extsearch.t
@@ -118,6 +118,26 @@ 	my $x2 = $es->over->get_xref3(2);
 	is(scalar(@$x1), 1, 'original only has one xref3');
 	is(scalar(@$x2), 1, 'new message has one xref3');
 	isnt($x1->[0], $x2->[0], 'xref3 differs');
+
+	my $mset = $es->mset('b:"BEST MSG"');
+	is($mset->size, 1, 'new message found');
+	$mset = $es->mset('b:"test message"');
+	is($mset->size, 1, 'old message found');
+
+	delete @$es{qw(git over xdb)}; # fork preparation
+
+	open my $rmfh, '+>', undef or BAIL_OUT $!;
+	$rmfh->autoflush(1);
+	print $rmfh $eml2->as_string or BAIL_OUT $!;
+	seek($rmfh, 0, SEEK_SET) or BAIL_OUT $!;
+	$opt->{0} = $rmfh;
+	ok(run_script([qw(-learn rm --all)], undef, $opt), '-learn rm');
+
+	ok(run_script([qw(-extindex --all), "$home/extindex"], undef, undef),
+		'extindex after rm');
+	is($es->over->get_art(2), undef, 'doc #2 gone');
+	$mset = $es->mset('b:"BEST MSG"');
+	is($mset->size, 0, 'new message gone');
 }
 
 my $misc = $es->misc;
diff --git a/t/over.t b/t/over.t
index 56c20d01799ce7a244eaa82fc3abb13b524efebe..22061249e6370e1e79969a51add351416e275817 100644
--- a/t/over.t
+++ b/t/over.t
@@ -91,7 +91,7 @@ 			'example.kee:2018:deadbeefcafe' ],
 			'xref3 works forw two');
 
 	@arg = qw(1349 adeadba7cafe example.key);
-	ok($over->remove_xref3(@arg), 'remove first');
+	is($over->remove_xref3(@arg), 1, 'remove first');
 	$xref3 = $over->get_xref3(1349);
 	is_deeply($xref3, [ 'example.kee:2018:deadbeefcafe' ],
 		'confirm removal successful');