lib/PublicInbox/LeiStore.pm | 54 ++++++++++++++++++++++++++++++++++------------------- lib/PublicInbox/SearchIdx.pm | 2 +- diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm index f978288a81612985f97673b520c9a1caf4c36016..4ba1e647dde82afd9c39f6c31c841216182e5923 100644 --- a/lib/PublicInbox/LeiStore.pm +++ b/lib/PublicInbox/LeiStore.pm @@ -226,6 +226,18 @@ my ($bref, $oidhex, $type, $size, $self) = @_; $self->{im}->remove($bref) if $bref; } +sub remove_docids ($;@) { + my ($self, @docids) = @_; + my $eidx = eidx_init($self); + for my $docid (@docids) { + $eidx->idx_shard($docid)->ipc_do('xdb_remove', $docid); + $self->{oidx}->delete_by_num($docid); + $self->{oidx}->{dbh}->do(<cat_async($oidhex, \&_remove_if_local, $self); } - $eidx->idx_shard($docid)->ipc_do('xdb_remove', $docid); - $oidx->delete_by_num($docid); } $git->cat_async_wait; + remove_docids($self, @docids); \@docids; } +sub oid2docid ($$) { + my ($self, $oid) = @_; + my $eidx = eidx_init($self); + my ($docid, @cull) = $eidx->{oidx}->blob_exists($oid); + if (@cull) { # fixup old bugs... + warn <{-fake_im} // $self->importer; # may create new epoch @@ -268,7 +292,7 @@ delete $xoids->{$smsg->{blob}}; # added later if (scalar keys %$xoids) { my %docids = map { $_ => 1 } @$vivify_xvmd; for my $oid (keys %$xoids) { - my @id = $oidx->blob_exists($oid); + my @id = oid2docid($self, $oid); @docids{@id} = @id; } @$vivify_xvmd = sort { $a <=> $b } keys(%docids); @@ -356,15 +380,11 @@ my ($eidx, $tl) = eidx_init($self); my $oidx = $eidx->{oidx}; my %seen; for my $oid (keys %$xoids) { - my @docids = $oidx->blob_exists($oid) or next; - scalar(@docids) > 1 and - warn "W: $oid indexed as multiple docids: @docids\n"; - for my $docid (@docids) { - next if $seen{$docid}++; - my $idx = $eidx->idx_shard($docid); - $idx->ipc_do('update_vmd', $docid, $vmd_mod); - } + my $docid = oid2docid($self, $oid) // next; delete $xoids->{$oid}; + next if $seen{$docid}++; + my $idx = $eidx->idx_shard($docid); + $idx->ipc_do('update_vmd', $docid, $vmd_mod); } return unless scalar(keys(%$xoids)); @@ -395,15 +415,11 @@ my %seen; # see if we can just update existing docs for my $oid (keys %$xoids) { - my @docids = $oidx->blob_exists($oid) or next; - scalar(@docids) > 1 and - warn "W: $oid indexed as multiple docids: @docids\n"; - for my $docid (@docids) { - next if $seen{$docid}++; - my $idx = $eidx->idx_shard($docid); - $idx->ipc_do('set_vmd', $docid, $vmd); - } + my $docid = oid2docid($self, $oid) // next; delete $xoids->{$oid}; # all done with this oid + next if $seen{$docid}++; + my $idx = $eidx->idx_shard($docid); + $idx->ipc_do('set_vmd', $docid, $vmd); } return unless scalar(keys(%$xoids)); diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index f066cc9285513b29c3f616d92516bf4647b93ab5..f553eda69b42d3ec8dfe23142812afe15d4f4104 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -572,7 +572,7 @@ my ($doc, $vmd_mod) = @_; my $updated = 0; my @x = @VMD_MAP; while (my ($field, $pfx) = splice(@x, 0, 2)) { - # field: "label" or "kw" + # field: "L" or "kw" for my $val (@{$vmd_mod->{"-$field"} // []}) { eval { $doc->remove_term($pfx . $val);